Add wider OpenGL coverage, more cleanups, etc..

Testing with some GLES 2.0 programs that I have made so I can profile
this thing.

3 files changed, 202 insertions, 103 deletions

diff --git a/glcapture.c b/glcapture.c
index 8db96fe..d620fe4 100644
--- a/glcapture.c
+++ b/glcapture.c
@@ -8,17 +8,14 @@
  * ^ Compile this branch of ffmpeg to get rawmux decoder
  * You can test that it works by doing ./ffplay /tmp/glcapture.fifo
  *
- * Make sure you increase your maximum pipe size /prox/sys/fs/pipe-max-size
- * to minimum of (FPS / 4) * ((width * height * 3) + 13)
+ * Make sure you increase your maximum pipe size /prox/sys/fs/pipe-max-size to minimum of
+ * (FPS / 4) * ((width * height * components) + 13) where components is 3 on OpenGL and 4 on OpenGL ES.
  *
  * If you get xruns from alsa, consider increasing your audio buffer size.
  */
 
 #define _GNU_SOURCE
 #include <dlfcn.h>
-#include <GL/glx.h>
-#include <EGL/egl.h>
-#include <alsa/asoundlib.h>
 #include <stdlib.h>
 #include <string.h>
 #include <stdio.h>
@@ -32,6 +29,10 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#include <GL/glx.h>
+#include <EGL/egl.h>
+#include <alsa/asoundlib.h>
+
 // Some tunables
 // XXX: Make these configurable
 
@@ -58,6 +59,26 @@ enum stream {
    STREAM_LAST,
 };
 
+// Set to false to disable stream
+static const bool ENABLED_STREAMS[STREAM_LAST] = {
+   true, // STREAM_VIDEO
+   true, // STREAM_AUDIO
+};
+
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
+#define WARN(x, ...) do { warn("glcapture: "x, ##__VA_ARGS__); } while (0)
+#define WARNX(x, ...) do { warnx("glcapture: "x, ##__VA_ARGS__); } while (0)
+#define ERRX(x, y, ...) do { errx(x, "glcapture: "y, ##__VA_ARGS__); } while (0)
+#define WARN_ONCE(x, ...) do { static bool o = false; if (!o) { WARNX(x, ##__VA_ARGS__); o = true; } } while (0)
+
+// "entrypoints" exposed to hooks.h
+static void swap_buffers(void);
+static void alsa_writei(snd_pcm_t *pcm, const void *buffer, const snd_pcm_uframes_t size, const char *caller);
+static uint64_t get_fake_time_ns(void);
+
+#include "hooks.h"
+#include "glwrangle.h"
+
 struct pbo {
    uint64_t ts;
    uint32_t width, height;
@@ -73,15 +94,15 @@ struct gl {
 struct frame_info {
    union {
       struct {
-         const char *format;
          uint32_t width, height, fps;
       } video;
       struct {
-         const char *format;
          uint32_t rate;
          uint8_t channels;
       } audio;
    };
+
+   const char *format;
    uint64_t ts;
    enum stream stream;
 };
@@ -89,28 +110,14 @@ struct frame_info {
 struct fifo {
    struct {
       struct frame_info info;
-      bool ready;
    } stream[STREAM_LAST];
+
    uint64_t base;
    size_t size;
    int fd;
    bool created;
 };
 
-#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
-#define WARN(x, ...) do { warn("glcapture: "x, ##__VA_ARGS__); } while (0)
-#define WARNX(x, ...) do { warnx("glcapture: "x, ##__VA_ARGS__); } while (0)
-#define ERRX(x, y, ...) do { errx(x, "glcapture: "y, ##__VA_ARGS__); } while (0)
-#define WARN_ONCE(x, ...) do { static bool o = false; if (!o) { WARNX(x, ##__VA_ARGS__); o = true; } } while (0)
-
-// "entrypoints" exposed to hooks.h
-static void swap_buffers(void);
-static void alsa_writei(snd_pcm_t *pcm, const void *buffer, const snd_pcm_uframes_t size, const char *caller);
-static uint64_t get_fake_time_ns(void);
-
-#include "hooks.h"
-#include "glwrangle.h"
-
 static uint64_t
 get_time_ns(void)
 {
@@ -134,8 +141,11 @@ write_rawmux_header(struct fifo *fifo)
 {
    uint8_t header[255] = { 'r', 'a', 'w', 'm', 'u', 'x' };
 
-   if (strlen(fifo->stream[STREAM_VIDEO].info.video.format) +
-       strlen(fifo->stream[STREAM_AUDIO].info.audio.format) + 33 > sizeof(header)) {
+   size_t variable_sz = 0;
+   for (enum stream i = 0; i < STREAM_LAST; ++i)
+      variable_sz += (fifo->stream[i].info.format ? strlen(fifo->stream[i].info.format) : 0);
+
+   if (variable_sz + 33 > sizeof(header)) {
       warnx("something went wrong");
       reset_fifo(fifo);
       return false;
@@ -144,20 +154,20 @@ write_rawmux_header(struct fifo *fifo)
    uint8_t *p = header + 6;
    memcpy(p, (uint8_t[]){1}, sizeof(uint8_t)); p += 1;
 
-   {
+   if (fifo->stream[STREAM_VIDEO].info.format) {
       const struct frame_info *info = &fifo->stream[STREAM_VIDEO].info;
       memcpy(p, (uint8_t[]){1}, sizeof(uint8_t)); p += 1;
-      memcpy(p, info->video.format, strlen(info->video.format)); p += strlen(info->video.format) + 1;
+      memcpy(p, info->format, strlen(info->format)); p += strlen(info->format) + 1;
       memcpy(p, (uint32_t[]){1}, sizeof(uint32_t)); p += 4;
       memcpy(p, (uint32_t[]){info->video.fps * 1000}, sizeof(uint32_t)); p += 4;
       memcpy(p, &info->video.width, sizeof(uint32_t)); p += 4;
       memcpy(p, &info->video.height, sizeof(uint32_t)); p += 4;
    }
 
-   {
+   if (fifo->stream[STREAM_AUDIO].info.format) {
       const struct frame_info *info = &fifo->stream[STREAM_AUDIO].info;
       memcpy(p, (uint8_t[]){2}, sizeof(uint8_t)); p += 1;
-      memcpy(p, info->audio.format, strlen(info->audio.format)); p += strlen(info->audio.format) + 1;
+      memcpy(p, info->format, strlen(info->format)); p += strlen(info->format) + 1;
       memcpy(p, &info->audio.rate, sizeof(info->audio.rate)); p += 4;
       memcpy(p, &info->audio.channels, sizeof(info->audio.channels)); p += 1;
    }
@@ -171,12 +181,12 @@ stream_info_changed(const struct frame_info *current, const struct frame_info *l
    assert(current->stream == last->stream);
 
    if (current->stream == STREAM_VIDEO) {
-      return (current->video.format != last->video.format ||
+      return (current->format != last->format ||
               current->video.width != last->video.width ||
               current->video.height != last->video.height);
    }
 
-   return (current->audio.format != last->audio.format ||
+   return (current->format != last->format ||
            current->audio.rate != last->audio.rate ||
            current->audio.channels != last->audio.channels);
 }
@@ -184,18 +194,15 @@ stream_info_changed(const struct frame_info *current, const struct frame_info *l
 static bool
 check_and_prepare_stream(struct fifo *fifo, const struct frame_info *info)
 {
-   if (fifo->stream[info->stream].ready && stream_info_changed(info, &fifo->stream[info->stream].info)) {
+   if (!ENABLED_STREAMS[info->stream])
+      return false;
+
+   if (fifo->stream[info->stream].info.format && stream_info_changed(info, &fifo->stream[info->stream].info)) {
       WARNX("stream information has changed");
       reset_fifo(fifo);
    }
 
    fifo->stream[info->stream].info = *info;
-   fifo->stream[info->stream].ready = true;
-
-   for (enum stream i = 0; i < STREAM_LAST; ++i) {
-      if (!fifo->stream[i].ready)
-         return false;
-   }
 
    if (!fifo->created) {
       remove(FIFO_PATH);
@@ -284,21 +291,42 @@ write_data(const struct frame_info *info, const void *buffer, const size_t size)
 static void
 capture_frame_pbo(struct gl *gl, const GLint view[4], const uint64_t ts)
 {
+   const struct {
+      const char *video;
+      GLenum format;
+      uint8_t components;
+   } frame = {
+      // XXX: Maybe on ES we should instead modify the data and remove A component?
+      //      Would save some transmission bandwidth at least
+      .video = (OPENGL_VARIANT == OPENGL_ES ? "rgba" : "rgb"),
+      .format = (OPENGL_VARIANT == OPENGL_ES ? GL_RGBA : GL_RGB),
+      .components = (OPENGL_VARIANT == OPENGL_ES ? 4 : 3),
+   };
+
    if (!glIsBuffer(gl->pbo[gl->active].obj)) {
       WARNX("create pbo %u", gl->active);
       glGenBuffers(1, &gl->pbo[gl->active].obj);
    }
 
    glBindBuffer(GL_PIXEL_PACK_BUFFER, gl->pbo[gl->active].obj);
-   glBufferData(GL_PIXEL_PACK_BUFFER, view[2] * view[3] * 3, NULL, GL_STREAM_READ);
+   glBufferData(GL_PIXEL_PACK_BUFFER, view[2] * view[3] * frame.components, NULL, GL_STREAM_READ);
+
+   struct { GLenum t; GLint o; GLint v; } map[] = {
+      { .t = GL_PACK_ALIGNMENT, .v = 1 },
+      { .t = GL_PACK_ROW_LENGTH },
+      { .t = GL_PACK_IMAGE_HEIGHT },
+      { .t = GL_PACK_SKIP_PIXELS },
+   };
+
+   for (size_t i = 0; i < ARRAY_SIZE(map); ++i) {
+      glGetIntegerv(map[i].t, &map[i].o);
+      glPixelStorei(map[i].t, map[i].v);
+   }
 
-   glPushClientAttrib(GL_CLIENT_PIXEL_STORE_BIT);
-   glPixelStorei(GL_PACK_ALIGNMENT, 1);
-   glPixelStorei(GL_PACK_ROW_LENGTH, 0);
-   glPixelStorei(GL_PACK_IMAGE_HEIGHT, 0);
-   glPixelStorei(GL_PACK_SKIP_PIXELS, 0);
-   glReadPixels(view[0], view[1], view[2], view[3], GL_RGB, GL_UNSIGNED_BYTE, NULL);
-   glPopClientAttrib();
+   glReadPixels(view[0], view[1], view[2], view[3], frame.format, GL_UNSIGNED_BYTE, NULL);
+
+   for (size_t i = 0; i < ARRAY_SIZE(map); ++i)
+      glPixelStorei(map[i].t, map[i].o);
 
    gl->pbo[gl->active].ts = ts;
    gl->pbo[gl->active].width = view[2];
@@ -308,19 +336,20 @@ capture_frame_pbo(struct gl *gl, const GLint view[4], const uint64_t ts)
    gl->active = (gl->active + 1) % NUM_PBOS;
 
    if (glIsBuffer(gl->pbo[gl->active].obj) && gl->pbo[gl->active].written) {
+      const struct frame_info info = {
+         .ts = gl->pbo[gl->active].ts,
+         .stream = STREAM_VIDEO,
+         .format = frame.video,
+         .video.width = gl->pbo[gl->active].width,
+         .video.height = gl->pbo[gl->active].height,
+         .video.fps = FPS,
+      };
+
       const void *buf;
+      const size_t size = info.video.width * info.video.height * frame.components;
       glBindBuffer(GL_PIXEL_PACK_BUFFER, gl->pbo[gl->active].obj);
-      if ((buf = glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY))) {
-         const struct frame_info info = {
-            .ts = gl->pbo[gl->active].ts,
-            .stream = STREAM_VIDEO,
-            .video.format = "rgb24",
-            .video.width = gl->pbo[gl->active].width,
-            .video.height = gl->pbo[gl->active].height,
-            .video.fps = FPS,
-         };
-
-         write_data(&info, buf, info.video.width * info.video.height * 3);
+      if ((buf = glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, size, GL_MAP_READ_BIT))) {
+         write_data(&info, buf, size);
          glUnmapBuffer(GL_PIXEL_PACK_BUFFER);
          gl->pbo[gl->active].written = false;
       }
@@ -331,11 +360,8 @@ static void
 reset_capture(struct gl *gl)
 {
    for (size_t i = 0; i < NUM_PBOS; ++i) {
-      if (glIsBuffer(gl->pbo[i].obj)) {
+      if (glIsBuffer(gl->pbo[i].obj))
          glDeleteBuffers(1, &gl->pbo[i].obj);
-      } else {
-         WARNX("seems like program recreated opengl context?");
-      }
    }
 
    WARNX("capture reset");
@@ -364,17 +390,26 @@ capture_frame(struct gl *gl, const GLint view[4])
 static void
 draw_indicator(const GLint view[4])
 {
+   GLfloat clear[4];
+   GLboolean scissor;
+   glGetFloatv(GL_COLOR_CLEAR_VALUE, clear);
+   glGetBooleanv(GL_SCISSOR_TEST, &scissor);
+
+   if (!scissor)
+      glEnable(GL_SCISSOR_TEST);
+
    const uint32_t size = (view[3] / 75 > 10 ? view[3] / 75 : 10);
-   glPushAttrib(GL_ENABLE_BIT | GL_COLOR_BUFFER_BIT | GL_SCISSOR_BIT);
-   glEnable(GL_SCISSOR_TEST);
    glScissor(size / 2 - 1, view[3] - size - size / 2 - 1, size + 2, size + 2);
    glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
    glClear(GL_COLOR_BUFFER_BIT);
    glScissor(size / 2, view[3] - size - size / 2, size, size);
    glClearColor(1.0f, 0.0f, 0.0f, 0.0f);
    glClear(GL_COLOR_BUFFER_BIT);
-   glDisable(GL_SCISSOR_TEST);
-   glPopAttrib();
+
+   if (!scissor)
+      glDisable(GL_SCISSOR_TEST);
+
+   glClearColor(clear[0], clear[1], clear[2], clear[3]);
 }
 
 static void
@@ -445,10 +480,10 @@ alsa_get_frame_info(snd_pcm_t *pcm, struct frame_info *out_info, const char *cal
    WARN_ONCE("%s (%s:%u:%u)", caller, snd_pcm_format_name(format), rate, channels);
    out_info->ts = get_time_ns();
    out_info->stream = STREAM_AUDIO;
-   out_info->audio.format = alsa_get_format(format);
+   out_info->format = alsa_get_format(format);
    out_info->audio.rate = rate;
    out_info->audio.channels = channels;
-   return (out_info->audio.format != NULL);
+   return (out_info->format != NULL);
 }
 
 static void
diff --git a/glwrangle.h b/glwrangle.h
index f6b2555..3da96b7 100644
--- a/glwrangle.h
+++ b/glwrangle.h
@@ -2,45 +2,71 @@
 
 static GLenum (*_glGetError)(void);
 static void (*_glGetIntegerv)(GLenum, GLint*);
+static void (*_glGetFloatv)(GLenum, GLfloat*);
+static void (*_glGetBooleanv)(GLenum, GLboolean*);
+static const char* (*_glGetString)(GLenum);
 static GLboolean (*_glIsBuffer)(GLuint);
 static void (*_glGenBuffers)(GLsizei, GLuint*);
 static void (*_glDeleteBuffers)(GLsizei, GLuint*);
 static void (*_glBindBuffer)(GLenum, GLuint);
 static void (*_glBufferData)(GLenum, GLsizeiptr, const GLvoid*, GLenum);
-static void* (*_glMapBuffer)(GLenum, GLenum);
+static void* (*_glMapBufferRange)(GLenum, GLintptr, GLsizeiptr, GLbitfield);
 static void (*_glUnmapBuffer)(GLenum);
 static void (*_glPixelStorei)(GLenum, GLint);
 static void (*_glReadPixels)(GLint, GLint, GLsizei, GLsizei, GLenum, GLenum, GLvoid*);
-static void (*_glPushClientAttrib)(GLbitfield);
-static void (*_glPopClientAttrib)(void);
-static void (*_glPushAttrib)(GLbitfield);
-static void (*_glPopAttrib)(void);
 static void (*_glEnable)(GLenum);
 static void (*_glDisable)(GLenum);
 static void (*_glScissor)(GLint, GLint, GLsizei, GLsizei);
 static void (*_glClearColor)(GLclampf, GLclampf, GLclampf, GLclampf);
 static void (*_glClear)(GLbitfield);
+static void (*_glDebugMessageCallback)(GLDEBUGPROC, const void*);
+
+enum gl_variant {
+   OPENGL_ES,
+   OPENGL,
+};
+
+struct gl_version {
+   uint32_t major, minor;
+};
+
+static enum gl_variant OPENGL_VARIANT;
+static struct gl_version OPENGL_VERSION;
 
 #define glGetError _glGetError
 #define glGetIntegerv _glGetIntegerv
+#define glGetFloatv _glGetFloatv
+#define glGetBooleanv _glGetBooleanv
+#define glGetString _glGetString
 #define glIsBuffer _glIsBuffer
 #define glGenBuffers _glGenBuffers
 #define glDeleteBuffers _glDeleteBuffers
 #define glBindBuffer _glBindBuffer
 #define glBufferData _glBufferData
-#define glMapBuffer _glMapBuffer
+#define glMapBufferRange _glMapBufferRange
 #define glUnmapBuffer _glUnmapBuffer
 #define glPixelStorei _glPixelStorei
 #define glReadPixels _glReadPixels
-#define glPushClientAttrib _glPushClientAttrib
-#define glPopClientAttrib _glPopClientAttrib
-#define glPushAttrib _glPushAttrib
-#define glPopAttrib _glPopAttrib
 #define glEnable _glEnable
 #define glDisable _glDisable
 #define glScissor _glScissor
 #define glClearColor _glClearColor
 #define glClear _glClear
+#define glDebugMessageCallback _glDebugMessageCallback
+
+static void
+debug_cb(GLenum source, GLenum type, GLuint id, GLenum severity, GLsizei length, const GLchar *message, const void *data)
+{
+   (void)source, (void)type, (void)id, (void)severity, (void)length,  (void)message, (void)data;
+   WARNX("%s", message);
+}
+
+static void*
+dlsym_proc(const char *procname)
+{
+   HOOK_DLSYM(dlsym);
+   return _dlsym(RTLD_NEXT, procname);
+}
 
 static void
 load_gl_function_pointers(void* (*procs[])(const char*), const size_t memb)
@@ -57,30 +83,70 @@ load_gl_function_pointers(void* (*procs[])(const char*), const size_t memb)
    }
 
    if (!proc)
-      ERRX(EXIT_FAILURE, "There is no proc loader available");
-
-#define GL(x) do { if (!(_##x = proc(#x))) { ERRX(EXIT_FAILURE, "Failed to load %s", #x); } } while (0)
-   GL(glGetError);
-   GL(glGetIntegerv);
-   GL(glIsBuffer);
-   GL(glGenBuffers);
-   GL(glDeleteBuffers);
-   GL(glBindBuffer);
-   GL(glBufferData);
-   GL(glMapBuffer);
-   GL(glUnmapBuffer);
-   GL(glPixelStorei);
-   GL(glReadPixels);
-   GL(glPushClientAttrib);
-   GL(glPopClientAttrib);
-   GL(glPushAttrib);
-   GL(glPopAttrib);
-   GL(glEnable);
-   GL(glDisable);
-   GL(glScissor);
-   GL(glClearColor);
-   GL(glClear);
+      proc = dlsym_proc;
+
+   // We try to support wide range of OpenGL and variants as possible.
+   // Thus avoid using functions that only work in certain OpenGL versions.
+   // e.g. glPushAttrib, glPushClientAttrib, it's bit of shitty but such is life.
+   // Alternatively if code starts getting too much saving/restoring, consider hooking
+   // the gl state changes we care about and write our own push/pop around swap_buffer.
+   //
+   // Version / variant dependant code is still possible through GL_VARIANT and GL_VERSION variables.
+   //
+   // Note that we also rely on system GL/glx.h for typedefs / constants, which probably is plain wrong on ES
+   // for example, but seems to work fine so far. Main interest is to work with mainly GLX / Wine games anyways.
+
+#define GL_REQUIRED(x) do { if (!(_##x = proc(#x))) { ERRX(EXIT_FAILURE, "Failed to load %s", #x); } } while (0)
+#define GL_OPTIONAL(x) do { _##x = proc(#x); } while (0)
+   GL_REQUIRED(glGetError);
+   GL_REQUIRED(glGetIntegerv);
+   GL_REQUIRED(glGetFloatv);
+   GL_REQUIRED(glGetBooleanv);
+   GL_REQUIRED(glGetString);
+   GL_REQUIRED(glIsBuffer);
+   GL_REQUIRED(glGenBuffers);
+   GL_REQUIRED(glDeleteBuffers);
+   GL_REQUIRED(glBindBuffer);
+   GL_REQUIRED(glBufferData);
+   GL_REQUIRED(glMapBufferRange);
+   GL_REQUIRED(glUnmapBuffer);
+   GL_REQUIRED(glPixelStorei);
+   GL_REQUIRED(glReadPixels);
+   GL_REQUIRED(glEnable);
+   GL_REQUIRED(glDisable);
+   GL_REQUIRED(glScissor);
+   GL_REQUIRED(glClearColor);
+   GL_REQUIRED(glClear);
+   GL_OPTIONAL(glDebugMessageCallback);
 #undef GL
 
+   if (glDebugMessageCallback) {
+      // GL_DEBUG_OUTPUT_SYNCHRONOUS for breakpoints (slower)
+      glEnable(GL_DEBUG_OUTPUT);
+      // glEnable(GL_DEBUG_OUTPUT_SYNCHRONOUS);
+      glDebugMessageCallback(debug_cb, NULL);
+   }
+
+   const struct { const char *p; enum gl_variant v; } variants[] = {
+      { .p = "OpenGL ES-CM ", .v = OPENGL_ES },
+      { .p = "OpenGL ES-CL ", .v = OPENGL_ES },
+      { .p = "OpenGL ES ", .v = OPENGL_ES },
+      { .p = "OpenGL ", .v = OPENGL },
+   };
+
+   const char *version = glGetString(GL_VERSION);
+   WARNX("%s", version);
+
+   for (size_t i = 0; i < ARRAY_SIZE(variants); ++i) {
+      const size_t len = strlen(variants[i].p);
+      if (strncmp(version, variants[i].p, len))
+         continue;
+
+      OPENGL_VARIANT = variants[i].v;
+      version += len;
+      break;
+   }
+
+   sscanf(version, "%u.%u", &OPENGL_VERSION.major, &OPENGL_VERSION.minor);
    loaded = true;
 }
diff --git a/hooks.h b/hooks.h
index b844fcd..6d91b8e 100644
--- a/hooks.h
+++ b/hooks.h
@@ -134,8 +134,8 @@ hook_function(void **ptr, const char *name, const bool versioned)
       return;
 
    if (versioned) {
-      const char *versions[] = { "GLIBC_2.0", "GLIBC_2.2.5", NULL };
-      for (size_t i = 0; !*ptr && versions[i]; ++i)
+      const char *versions[] = { "GLIBC_2.0", "GLIBC_2.2.5" };
+      for (size_t i = 0; !*ptr && i < ARRAY_SIZE(versions); ++i)
          *ptr = dlvsym(RTLD_NEXT, name, versions[i]);
    } else {
       HOOK_DLSYM(dlsym);
@@ -158,5 +158,3 @@ dlsym(void *handle, const char *symbol)
 
    return store_real_symbol_and_return_fake_symbol(symbol, _dlsym(handle, symbol));
 }
-
-#undef HOOK_DLSYM