etnaviv: move fence handling to libetnaviv

This makes fence handles into globally increasing uint32_t's
and thus simplifies the handling in the driver, and makes it possible
to compare fence handles to see what happened before.
The scheme emulates a saner kernel API (closer to adreno drm driver)

19 files changed, 301 insertions, 248 deletions

diff --git a/src/driver/etna_fence.c b/src/driver/etna_fence.c
index 018f67a..ecedee8 100644
--- a/src/driver/etna_fence.c
+++ b/src/driver/etna_fence.c
@@ -9,116 +9,17 @@
 #include <etnaviv/etna.h>
 #include <etnaviv/etna_queue.h>
 
-static void etna_screen_fence_reference(struct pipe_screen *screen_h,
-                        struct pipe_fence_handle **ptr_h,
-                        struct pipe_fence_handle *fence_h );
-static boolean etna_screen_fence_signalled(struct pipe_screen *screen_h,
-                           struct pipe_fence_handle *fence_h);
-static boolean etna_screen_fence_finish(struct pipe_screen *screen_h,
-                        struct pipe_fence_handle *fence_h,
-                        uint64_t timeout );
-
-int etna_fence_new(struct pipe_screen *screen_h, struct etna_ctx *ctx, struct pipe_fence_handle **fence_p)
-{
-    struct etna_fence *fence = NULL;
-    struct etna_screen *screen = etna_screen(screen_h);
-    int rv;
-
-    /* XXX we do not release the fence_p reference here -- neither do the other drivers,
-     * and clients don't seem to rely on this. */
-    if(fence_p == NULL)
-        return ETNA_INVALID_ADDR;
-    assert(*fence_p == NULL);
-
-    /* re-use old fence, if available, and reset it first */
-    pipe_mutex_lock(screen->fence_mutex);
-    if(screen->fence_freelist != NULL)
-    {
-        fence = screen->fence_freelist;
-        screen->fence_freelist = fence->next_free;
-        fence->next_free = NULL;
-    }
-    pipe_mutex_unlock(screen->fence_mutex);
-
-    if(fence != NULL)
-    {
-        if((rv = viv_user_signal_signal(ctx->conn, fence->signal, 0)) != VIV_STATUS_OK)
-        {
-            BUG("Error: could not reset signal %i", fence->signal);
-            etna_screen_destroy_fence(screen_h, fence);
-            return rv;
-        }
-        fence->signalled = false;
-    } else {
-        fence = CALLOC_STRUCT(etna_fence);
-        /* Create signal with manual reset; we want to be able to probe it
-         * or wait for it without resetting it.
-         */
-        if((rv = viv_user_signal_create(ctx->conn, /* manualReset */ true, &fence->signal)) != VIV_STATUS_OK)
-        {
-            FREE(fence);
-            return rv;
-        }
-    }
-    if((rv = etna_queue_signal(ctx->queue, fence->signal, VIV_WHERE_PIXEL)) != ETNA_OK)
-    {
-        BUG("error queueing signal %i", fence->signal);
-        viv_user_signal_destroy(ctx->conn, fence->signal);
-        FREE(fence);
-        return rv;
-    }
-    pipe_reference_init(&fence->reference, 1);
-    *fence_p = (struct pipe_fence_handle*)fence;
-    return ETNA_OK;
-}
-
-static void
-debug_describe_fence(char* buf, const struct etna_fence *fence)
-{
-    util_sprintf(buf, "etna_fence<%i>", fence->signal);
-}
-
 /**
- * Reference or unreference a fence. Once the reference count falls to zero,
- * the fence will be destroyed or put in the free list to be reused.
+ * Reference or unreference a fence. This is pretty much a no-op.
  */
 static void etna_screen_fence_reference(struct pipe_screen *screen_h,
                         struct pipe_fence_handle **ptr_h,
-                        struct pipe_fence_handle *fence_h )
+                        struct pipe_fence_handle *fence_h)
 {
-    struct etna_screen *screen = etna_screen(screen_h);
-    struct etna_fence *fence = etna_fence(fence_h);
-    struct etna_fence **ptr = (struct etna_fence **) ptr_h;
-    struct etna_fence *old_fence = *ptr;
-    if (pipe_reference_described(&(*ptr)->reference, &fence->reference,
-                                 (debug_reference_descriptor)debug_describe_fence))
-    {
-        if(etna_screen_fence_signalled(screen_h, (struct pipe_fence_handle*)old_fence))
-        {
-            /* If signalled, add old fence to free list, as it can be reused */
-            pipe_mutex_lock(screen->fence_mutex);
-            old_fence->next_free = screen->fence_freelist;
-            screen->fence_freelist = old_fence;
-            pipe_mutex_unlock(screen->fence_mutex);
-        } else {
-            /* If fence is still to be signalled, destroy it, to prevent it from being
-             * reused. */
-            etna_screen_destroy_fence(screen_h, old_fence);
-        }
-    }
     *ptr_h = fence_h;
 }
 
 /**
- * Poll whether the fence has been signalled.
- */
-static boolean etna_screen_fence_signalled(struct pipe_screen *screen_h,
-                           struct pipe_fence_handle *fence_h)
-{
-    return etna_screen_fence_finish(screen_h, fence_h, 0);
-}
-
-/**
  * Wait until the fence has been signalled for the specified timeout in nanoseconds,
  * or PIPE_TIMEOUT_INFINITE.
  */
@@ -127,43 +28,25 @@ static boolean etna_screen_fence_finish(struct pipe_screen *screen_h,
                         uint64_t timeout )
 {
     struct etna_screen *screen = etna_screen(screen_h);
-    struct etna_fence *fence = etna_fence(fence_h);
+    uint32_t fence = PIPE_HANDLE_TO_ETNA_FENCE(fence_h);
     int rv;
-    if(fence->signalled) /* avoid a kernel roundtrip */
-        return true;
     /* nanoseconds to milliseconds */
-    rv = viv_user_signal_wait(screen->dev, fence->signal,
+    rv = viv_fence_finish(screen->dev, fence,
             timeout == PIPE_TIMEOUT_INFINITE ? VIV_WAIT_INDEFINITE : (timeout / 1000000ULL));
     if(rv != VIV_STATUS_OK && rv != VIV_STATUS_TIMEOUT)
     {
-        BUG("error waiting for signal %i", fence->signal);
+        BUG("error waiting for fence %08x", fence);
     }
-    fence->signalled = (rv != VIV_STATUS_TIMEOUT);
-    return fence->signalled;
+    return (rv != VIV_STATUS_TIMEOUT);
 }
 
-void etna_screen_destroy_fence(struct pipe_screen *screen_h, struct etna_fence *fence)
-{
-    struct etna_screen *screen = etna_screen(screen_h);
-    if(viv_user_signal_destroy(screen->dev, fence->signal) != VIV_STATUS_OK)
-    {
-        BUG("cannot destroy signal %i", fence->signal);
-    }
-    FREE(fence);
-}
-
-void etna_screen_destroy_fences(struct pipe_screen *screen_h)
+/**
+ * Poll whether the fence has been signalled.
+ */
+static boolean etna_screen_fence_signalled(struct pipe_screen *screen_h,
+                           struct pipe_fence_handle *fence_h)
 {
-    struct etna_screen *screen = etna_screen(screen_h);
-    struct etna_fence *fence, *next;
-    pipe_mutex_lock(screen->fence_mutex);
-    for(fence = screen->fence_freelist; fence != NULL; fence = next)
-    {
-        next = fence->next_free;
-        etna_screen_destroy_fence(screen_h, fence);
-    }
-    screen->fence_freelist = NULL;
-    pipe_mutex_unlock(screen->fence_mutex);
+    return etna_screen_fence_finish(screen_h, fence_h, 0);
 }
 
 void etna_screen_fence_init(struct pipe_screen *pscreen)
diff --git a/src/driver/etna_fence.h b/src/driver/etna_fence.h
index 41afbc6..639a46f 100644
--- a/src/driver/etna_fence.h
+++ b/src/driver/etna_fence.h
@@ -27,42 +27,9 @@
 #include "pipe/p_state.h"
 
 struct pipe_screen;
-struct pipe_fence_handle;
-struct etna_ctx;
 
-struct etna_fence
-{
-    struct pipe_reference reference;
-    int signal; /* signal id from kernel */
-    bool signalled; /* cached value of signalled */
-    struct etna_fence *next_free; /* if in free list, reference to next free fence */
-};
-
-/** Convert generic pipe_fence_handle pointer to etna_fence */
-static INLINE struct etna_fence *
-etna_fence(struct pipe_fence_handle *pfence)
-{
-    return (struct etna_fence *)pfence;
-}
-
-/**
- * Create a new fence that will be signalled after GPU completes rendering
- * after the next flush.
- */
-int etna_fence_new(struct pipe_screen *screen,
-                   struct etna_ctx *ctx,
-                   struct pipe_fence_handle **fence);
-
-/**
- * Destroy a fence. In general, you should call etna_screen_fence_reference instead,
- * if there may be other references.
- */
-void etna_screen_destroy_fence(struct pipe_screen *screen_h, struct etna_fence *fence);
-
-/**
- * Destroy all fences kept around for re-use in the free list.
- */
-void etna_screen_destroy_fences(struct pipe_screen *screen_h);
+#define ETNA_FENCE_TO_PIPE_HANDLE(fence) ((struct pipe_fence_handle *)(fence))
+#define PIPE_HANDLE_TO_ETNA_FENCE(fence) ((uint32_t)(fence))
 
 void etna_screen_fence_init(struct pipe_screen *screen);
 
diff --git a/src/driver/etna_pipe.c b/src/driver/etna_pipe.c
index 9ba6c56..358c686 100644
--- a/src/driver/etna_pipe.c
+++ b/src/driver/etna_pipe.c
@@ -1231,21 +1231,19 @@ static void etna_pipe_set_index_buffer( struct pipe_context *pipe,
 }
 
 static void etna_pipe_flush(struct pipe_context *pipe,
-             struct pipe_fence_handle **fence,
+             struct pipe_fence_handle **fence_out,
              enum pipe_flush_flags flags)
 {
     struct etna_pipe_context *priv = etna_pipe_context(pipe);
-    if(fence)
-    {
-        if(etna_fence_new(pipe->screen, priv->ctx, fence) != ETNA_OK)
-        {
-            BUG("Error: could not create fence");
-        }
-    }
-    if(etna_flush(priv->ctx) != ETNA_OK)
+    uint32_t _fence_tmp; /* just pass through fence, though we have to convert the type... */
+    uint32_t *fence_in = (fence_out == NULL) ? NULL : (&_fence_tmp);
+    if(etna_flush(priv->ctx, fence_in) != ETNA_OK)
     {
         BUG("Error: etna_flush failed, GPU may be in unpredictable state");
     }
+    if(fence_out)
+        *fence_out = ETNA_FENCE_TO_PIPE_HANDLE(*fence_in);
+
     if(DBG_ENABLED(ETNA_DBG_FINISH_ALL))
     {
         if(etna_finish(priv->ctx) != ETNA_OK)
diff --git a/src/driver/etna_screen.c b/src/driver/etna_screen.c
index bfe2c27..9ba297b 100644
--- a/src/driver/etna_screen.c
+++ b/src/driver/etna_screen.c
@@ -81,9 +81,7 @@ static void etna_set_debug_flags(const char *str)
 
 static void etna_screen_destroy( struct pipe_screen *screen )
 {
-    struct etna_screen *priv = etna_screen(screen);
-    etna_screen_destroy_fences(screen);
-    pipe_mutex_destroy(priv->fence_mutex);
+    //struct etna_screen *priv = etna_screen(screen);
     FREE(screen);
 }
 
@@ -572,8 +570,6 @@ etna_screen_create(struct viv_conn *dev)
     etna_screen_fence_init(pscreen);
     etna_screen_resource_init(pscreen);
 
-    pipe_mutex_init(screen->fence_mutex);
-
     return pscreen;
 }
 
diff --git a/src/driver/etna_screen.h b/src/driver/etna_screen.h
index 75a88ec..880e55d 100644
--- a/src/driver/etna_screen.h
+++ b/src/driver/etna_screen.h
@@ -38,9 +38,6 @@ struct etna_screen {
     char name[ETNA_SCREEN_NAME_LEN];
     struct viv_conn *dev;
     struct etna_pipe_specs specs;
-
-    pipe_mutex fence_mutex;
-    struct etna_fence *fence_freelist;
 };
 
 /* Resolve target.
diff --git a/src/etnaviv/etna.c b/src/etnaviv/etna.c
index 93af9cb..cf15cda 100644
--- a/src/etnaviv/etna.c
+++ b/src/etnaviv/etna.c
@@ -451,7 +451,7 @@ int _etna_reserve_internal(struct etna_ctx *ctx, size_t n)
             abort(); /* buffer is in invalid state XXX need some kind of recovery */
         }
         /* Otherwise, if there is something to be committed left in the current command buffer, commit it */
-        if((status = etna_flush(ctx)) != ETNA_OK)
+        if((status = etna_flush(ctx, NULL)) != ETNA_OK)
         {
             printf("%s: reserve failed: %i\n", __func__, status);
             abort(); /* buffer is in invalid state XXX need some kind of recovery */
@@ -468,27 +468,72 @@ int _etna_reserve_internal(struct etna_ctx *ctx, size_t n)
     return status;
 }
 
-int etna_flush(struct etna_ctx *ctx)
+int etna_flush(struct etna_ctx *ctx, uint32_t *fence_out)
 {
-    int status;
+    int status = ETNA_OK;
     if(ctx == NULL)
         return ETNA_INVALID_ADDR;
     if(ctx->cur_buf == ETNA_CTX_BUFFER)
         /* Can never flush while building context buffer */
         return ETNA_INTERNAL_ERROR;
-    struct _gcsQUEUE *queue_first = etna_queue_first(ctx->queue);
-    if(ctx->cur_buf == ETNA_NO_BUFFER)
-        goto nothing_to_do;
-    gcoCMDBUF cur_buf = ctx->cmdbuf[ctx->cur_buf];
 
-    ETNA_ALIGN(ctx); /* make sure end of submitted command buffer end is aligned */
+    if(fence_out) /* is a fence handle requested? */
+    {
+        uint32_t fence;
+        int signal;
+        /* Need to lock the fence mutex to make sure submits are ordered by
+         * fence number.
+         */
+        pthread_mutex_lock(&ctx->conn->fence_mutex);
+        do {
+            /*   Get next fence ID */
+            if((status = _viv_fence_new(ctx->conn, &fence, &signal)) != VIV_STATUS_OK)
+            {
+                printf("%s: could not request fence\n", __func__);
+                goto unlock_and_return_status;
+            }
+        } while(fence == 0); /* don't return fence handle 0 as it is interpreted as error value downstream */
+        /*   Queue the signal. This can call in turn call this function (but
+         * without fence) if the queue was full, so we should be able to handle
+         * that. In that case, we will exit from this function with only
+         * this fence in the queue and an empty command buffer.
+         */
+        if((status = etna_queue_signal(ctx->queue, signal, VIV_WHERE_PIXEL)) != ETNA_OK)
+        {
+            printf("%s: error %i queueing fence signal %i\n", __func__, status, signal);
+            goto unlock_and_return_status;
+        }
+        *fence_out = fence;
+    }
+    /***** Start fence mutex locked */
+    /* Make sure to unlock the mutex before returning */
+    struct _gcsQUEUE *queue_first = _etna_queue_first(ctx->queue);
+    gcoCMDBUF cur_buf = (ctx->cur_buf != ETNA_NO_BUFFER) ? ctx->cmdbuf[ctx->cur_buf] : NULL;
+
+    if(cur_buf == NULL || (ctx->offset*4 <= (cur_buf->startOffset + BEGIN_COMMIT_CLEARANCE)))
+    {
+        /* Nothing in command buffer; but if we end up here there may be kernel commands to submit. Do this seperately. */
+        if(queue_first != NULL)
+        {
+            ctx->flushes = 0;
+            if((status = viv_event_commit(ctx->conn, queue_first)) != 0)
+            {
+#ifdef DEBUG
+                fprintf(stderr, "Error committing kernel commands\n");
+#endif
+                goto unlock_and_return_status;
+            }
+            if(fence_out) /* mark fence as submitted to kernel */
+                _viv_fence_mark_pending(ctx->conn, *fence_out);
+        }
+        goto unlock_and_return_status;
+    }
+
+    cur_buf->offset = ctx->offset*4; /* Copy over current end offset into CMDBUF, for kernel */
 #ifdef DEBUG
     printf("Committing command buffer %i startOffset=%x offset=%x\n", ctx->cur_buf,
             cur_buf->startOffset, ctx->offset*4);
 #endif
-    if(ctx->offset*4 <= (cur_buf->startOffset + BEGIN_COMMIT_CLEARANCE))
-        goto nothing_to_do;
-    cur_buf->offset = ctx->offset*4; /* Copy over current ending offset into CMDBUF, for kernel */
 #ifdef DEBUG_CMDBUF
     etna_dump_cmd_buffer(ctx);
 #endif
@@ -504,10 +549,14 @@ int etna_flush(struct etna_ctx *ctx)
 #ifdef DEBUG
         fprintf(stderr, "Error committing command buffer\n");
 #endif
-        return status;
+        goto unlock_and_return_status;
     }
-    if((status = etna_queue_clear(ctx->queue)) != ETNA_OK)
-        return status;
+    if(fence_out)
+    {
+        _viv_fence_mark_pending(ctx->conn, *fence_out);
+        pthread_mutex_unlock(&ctx->conn->fence_mutex);
+    }
+    /***** End fence mutex locked */
 #ifdef GCABI_HAS_CONTEXT
     /* set context entryPipe to currentPipe (next commit will start with current pipe) */
     GCCTX(ctx)->entryPipe = GCCTX(ctx)->currentPipe;
@@ -559,22 +608,11 @@ int etna_flush(struct etna_ctx *ctx)
 #endif
 #endif
     return ETNA_OK;
-nothing_to_do:
-    /* Nothing in command buffer; but there may be kernel commands to submit. Do this seperately. */
-    if(queue_first != NULL)
-    {
-        ctx->flushes = 0;
-        if((status = viv_event_commit(ctx->conn, queue_first)) != 0)
-        {
-#ifdef DEBUG
-            fprintf(stderr, "Error committing kernel commands\n");
-#endif
-            return status;
-        }
-        if((status = etna_queue_clear(ctx->queue)) != ETNA_OK)
-            return status;
-    }
-    return ETNA_OK;
+
+unlock_and_return_status: /* Unlock fence mutex (if necessary) and return status */
+    if(fence_out)
+        pthread_mutex_unlock(&ctx->conn->fence_mutex);
+    return status;
 }
 
 int etna_finish(struct etna_ctx *ctx)
@@ -587,7 +625,7 @@ int etna_finish(struct etna_ctx *ctx)
     {
         return ETNA_INTERNAL_ERROR;
     }
-    if((status = etna_flush(ctx)) != ETNA_OK)
+    if((status = etna_flush(ctx, NULL)) != ETNA_OK)
         return status;
 #ifdef DEBUG
     printf("finish: Waiting for signal...\n");
diff --git a/src/etnaviv/etna.h b/src/etnaviv/etna.h
index a4e3255..8fc51d8 100644
--- a/src/etnaviv/etna.h
+++ b/src/etnaviv/etna.h
@@ -229,7 +229,7 @@ int etna_set_pipe(struct etna_ctx *ctx, enum etna_pipe pipe);
 /* Send currently queued commands to kernel.
  * @return OK on success, error code otherwise
  */
-int etna_flush(struct etna_ctx *ctx);
+int etna_flush(struct etna_ctx *ctx, uint32_t *fence_out);
 
 /* Send currently queued commands to kernel, then block for them to finish.
  * @return OK on success, error code otherwise
diff --git a/src/etnaviv/etna_queue.c b/src/etnaviv/etna_queue.c
index c874087..a40b510 100644
--- a/src/etnaviv/etna_queue.c
+++ b/src/etnaviv/etna_queue.c
@@ -27,25 +27,28 @@ int etna_queue_create(struct etna_ctx *ctx, struct etna_queue **queue_out)
     return ETNA_OK;
 }
 
-int etna_queue_clear(struct etna_queue *queue)
+struct _gcsQUEUE *_etna_queue_first(struct etna_queue *queue)
 {
+    struct _gcsQUEUE *rv = (queue->count == 0) ? NULL : queue->queue;
     queue->last = NULL;
     queue->count = 0;
-    return ETNA_OK;
+    return rv;
 }
 
 int etna_queue_alloc(struct etna_queue *queue, struct _gcsHAL_INTERFACE **cmd_out)
 {
-    int rv;
     if(queue == NULL)
         return ETNA_INVALID_ADDR;
     if(queue->count == queue->max_count)
     {
+        int rv;
         /* Queue is full, flush context. Assert that there is a one-to-one relationship
          * between queue and etna context so that flushing the context flushes this queue.
+         *
+         * Don't request a fence to prevent an infinite loop.
          */
         assert(queue->ctx->queue == queue);
-        if((rv = etna_flush(queue->ctx)) != ETNA_OK)
+        if((rv = etna_flush(queue->ctx, NULL)) != ETNA_OK)
             return rv;
         assert(queue->count == 0);
     }
diff --git a/src/etnaviv/etna_queue.h b/src/etnaviv/etna_queue.h
index d9f0a1b..27f55aa 100644
--- a/src/etnaviv/etna_queue.h
+++ b/src/etnaviv/etna_queue.h
@@ -48,25 +48,12 @@ struct etna_queue {
  */
 int etna_queue_create(struct etna_ctx *ctx, struct etna_queue **queue_out);
 
-/* Clear the queue, removing all commands.
- */
-int etna_queue_clear(struct etna_queue *queue);
-
-/* Return pointer to first element in queue, or NULL if queue is empty.
+/* Return pointer to first element in queue, or NULL if queue is empty, and
+ * empty the queue so that it can be used for the next flush.
  * As the queue is submitted to the kernel as a linked list, a pointer to the first element is enough
  * to represent it.
- *
- * Submitting the queue is currently a two-step process:
- *
- * - Request first pointer using etna_queue_first
- * - Submit queue to kernel
- * - Clear queue using etna_queue_clear
- *
  */
-static inline struct _gcsQUEUE *etna_queue_first(struct etna_queue *queue)
-{
-    return (queue->count == 0) ? NULL : queue->queue;
-}
+struct _gcsQUEUE *_etna_queue_first(struct etna_queue *queue);
 
 /* Allocate a new kernel command structure, add it to the queue,
  * and return a pointer to it.
diff --git a/src/etnaviv/viv.c b/src/etnaviv/viv.c
index b9b400a..6bcfc13 100644
--- a/src/etnaviv/viv.c
+++ b/src/etnaviv/viv.c
@@ -50,6 +50,49 @@
 const char *galcore_device[] = {"/dev/gal3d", "/dev/galcore", "/dev/graphics/galcore", NULL};
 #define INTERFACE_SIZE (sizeof(gcsHAL_INTERFACE))
 
+/* Allocate signals for fences */
+static int viv_allocate_signals(struct viv_conn *conn)
+{
+    int rv;
+    if(pthread_mutex_init(&conn->fence_mutex, NULL))
+        return VIV_STATUS_OUT_OF_MEMORY;
+    for(int x=0; x<VIV_NUM_FENCE_SIGNALS; ++x)
+    {
+        /* Create signal with manual reset; we want to be able to probe it
+         * or wait for it without resetting it.
+         */
+        if((rv = viv_user_signal_create(conn, /* manualReset */ false, &conn->fence_signals[x])) != VIV_STATUS_OK)
+        {
+            return rv;
+        }
+    }
+    conn->next_fence_id = 0;
+    conn->last_fence_id = -1; /* far enough into the past */
+    return VIV_STATUS_OK;
+}
+
+/* Free signals for fences */
+static int viv_deallocate_signals(struct viv_conn *conn)
+{
+    int rv;
+    for(int x=0; x<VIV_NUM_FENCE_SIGNALS; ++x)
+    {
+        if((rv = viv_user_signal_destroy(conn, conn->fence_signals[x])) != VIV_STATUS_OK)
+            return rv;
+    }
+    if(pthread_mutex_destroy(&conn->fence_mutex))
+        return VIV_STATUS_OUT_OF_MEMORY;
+    return VIV_STATUS_OK;
+}
+
+/* Get signal id # for a fence */
+static int signal_for_fence(struct viv_conn *conn, uint32_t fence)
+{
+    if((conn->next_fence_id - fence) >= VIV_NUM_FENCE_SIGNALS)
+        return -1; /* too old */
+    return conn->fence_signals[fence % VIV_NUM_FENCE_SIGNALS];
+}
+
 /* Call ioctl interface with structure cmd as input and output.
  * @returns status (VIV_STATUS_xxx)
  */
@@ -86,6 +129,9 @@ int viv_close(struct viv_conn *conn)
 {
     if(conn->fd < 0)
         return -1;
+
+    (void) viv_deallocate_signals(conn);
+
     close(conn->fd);
     free(conn);
 #ifdef ETNAVIV_HOOK
@@ -217,6 +263,9 @@ int viv_open(enum viv_hw_type hw_type, struct viv_conn **out)
 
     conn->process = getpid(); /* value passed as .process to commands */
 
+    if((err=viv_allocate_signals(conn)) != VIV_STATUS_OK)
+        goto error;
+
     *out = conn;
     return gcvSTATUS_OK;
 error:
@@ -560,3 +609,98 @@ int viv_write_register(struct viv_conn *conn, uint32_t address, uint32_t data)
     return viv_invoke(conn, &id);
 }
 
+int _viv_fence_new(struct viv_conn *conn, uint32_t *fence_out, int *signal_out)
+{
+    /* Request fence and queue signal */
+    uint32_t fence = conn->next_fence_id++;
+    int signal = signal_for_fence(conn, fence);
+    int status;
+    /*   First, wait for old signal before reusing it if needed */
+    uint32_t oldfence = fence - VIV_NUM_FENCE_SIGNALS;
+    uint32_t fence_mod_signals = (fence % VIV_NUM_FENCE_SIGNALS);
+    if(conn->fences_pending & (1<<fence_mod_signals)) /* fence still pending? */
+    {
+#ifdef FENCE_DEBUG
+        printf("Waiting for old fence %08x (which is after %08x)\n", oldfence,
+                conn->last_fence_id);
+#endif
+        if((status = viv_user_signal_wait(conn, signal, VIV_WAIT_INDEFINITE)) != VIV_STATUS_OK)
+        {
+            return status;
+        }
+        /* update last signalled fence if necessary */
+        if(VIV_FENCE_BEFORE(conn->last_fence_id, oldfence))
+            conn->last_fence_id = oldfence;
+        conn->fences_pending &= ~(1<<fence_mod_signals);
+    }
+    *fence_out = fence;
+    *signal_out = signal;
+#ifdef FENCE_DEBUG
+    printf("New fence: %08x [signal %08x], pending %08x\n", fence, signal, conn->fences_pending);
+#endif
+    return VIV_STATUS_OK;
+}
+
+void _viv_fence_mark_pending(struct viv_conn *conn, uint32_t fence)
+{
+    if((conn->next_fence_id - fence) >= VIV_NUM_FENCE_SIGNALS)
+        return; /* too old */
+    conn->fences_pending |= (1<<(fence % VIV_NUM_FENCE_SIGNALS));
+}
+
+int viv_fence_finish(struct viv_conn *conn, uint32_t fence, uint32_t timeout)
+{
+    int signal;
+    int rv;
+    pthread_mutex_lock(&conn->fence_mutex);
+    signal = signal_for_fence(conn, fence);
+    if(signal == -1)
+    {
+#ifdef FENCE_DEBUG
+        printf("Fence already expired: %08x\n", fence);
+#endif
+        goto unlock_and_ok; /* fence too old, it must have been signalled already */
+    }
+    /* If fence is older than last_fence_id which is the last signalled fence,
+     * it must already have been signalled. We can make use of the fact that there is only
+     * one ringbuffer inside the kernel, so commands submitted prior to this
+     * fence will be executed before this fence.
+     * Also check whether fence is really pending, if not simply return.
+     */
+    uint32_t fence_mod_signals = (fence % VIV_NUM_FENCE_SIGNALS);
+    if(!(conn->fences_pending & (1<<fence_mod_signals)) ||
+        VIV_FENCE_BEFORE_EQ(fence, conn->last_fence_id))
+    {
+#ifdef FENCE_DEBUG
+        printf("Fence already signaled: %08x, pending %i, newer than %08x; next fence id is %08x\n",
+                fence,
+                (conn->fences_pending >> fence_mod_signals)&1,
+                conn->last_fence_id, conn->next_fence_id);
+#endif
+        goto unlock_and_ok;
+    }
+    pthread_mutex_unlock(&conn->fence_mutex); /* don't keep mutex while waiting */
+
+    rv = viv_user_signal_wait(conn, signal, timeout);
+    if(rv == VIV_STATUS_OK)
+    {
+        pthread_mutex_lock(&conn->fence_mutex);
+        /* mark fence as non-pending */
+        conn->fences_pending &= ~(1<<fence_mod_signals);
+        /* if fence is later than last_fence_id, update last_fence_id */
+        if(VIV_FENCE_BEFORE(conn->last_fence_id, fence))
+        {
+            conn->last_fence_id = fence;
+#ifdef FENCE_DEBUG
+            printf("Last fence id updated to %i\n", conn->last_fence_id);
+#endif
+        }
+        pthread_mutex_unlock(&conn->fence_mutex);
+    }
+    return rv;
+
+unlock_and_ok: /* unlock mutex and return OK */
+    pthread_mutex_unlock(&conn->fence_mutex);
+    return VIV_STATUS_OK;
+}
+
diff --git a/src/etnaviv/viv.h b/src/etnaviv/viv.h
index 4d0ebd4..8aaf01e 100644
--- a/src/etnaviv/viv.h
+++ b/src/etnaviv/viv.h
@@ -27,9 +27,18 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <stdbool.h>
+#include <pthread.h>
 
 #define VIV_WAIT_INDEFINITE (0xffffffff)
 
+/* Number of signals to keep for fences, max is 32 */
+#define VIV_NUM_FENCE_SIGNALS 32
+
+/* Return true if fence a was before b */
+#define VIV_FENCE_BEFORE(a,b) ((int32_t)((b)-(a))>0)
+/* Return true if fence a was before or equal to b */
+#define VIV_FENCE_BEFORE_EQ(a,b) ((int32_t)((b)-(a))>=0)
+
 /* Enum with indices for each of the feature words */
 enum viv_features_word
 {
@@ -184,6 +193,16 @@ struct viv_conn {
     viv_handle_t process;
     struct viv_specs chip;
     struct viv_kernel_driver_version kernel_driver;
+    /* signals for fences */
+    int fence_signals[VIV_NUM_FENCE_SIGNALS];
+    /* guard these with a mutex, so
+     * that no races happen and command buffers are submitted
+     * in the same order as the fence ids.
+     */
+    pthread_mutex_t fence_mutex;
+    uint32_t next_fence_id; /* Next fence number to be dealt */
+    uint32_t fences_pending; /* Bitmask of fences signalled but not yet waited for */
+    uint32_t last_fence_id; /* Most recent signalled fence */
 };
 
 /* Predefines for some kernel structures */
@@ -289,6 +308,27 @@ int viv_read_register(struct viv_conn *conn, uint32_t address, uint32_t *data);
  */
 int viv_write_register(struct viv_conn *conn, uint32_t address, uint32_t data);
 
+/** Internal: Request a new fence handle and return it. Also return signal id
+ * associated with the fence, to add to queue.
+ * @note must be called with fence_mutex held.
+ */
+int _viv_fence_new(struct viv_conn *conn, uint32_t *fence_out, int *signal_out);
+
+/** Internal: Mark a fence as pending.
+ * Call this only after submitting the signal to the kernel.
+ * @note must be called with fence_mutex held.
+ */
+void _viv_fence_mark_pending(struct viv_conn *conn, uint32_t fence);
+
+/** Wait for fence or poll status.
+ * Timeout is in milliseconds.
+ * Pass a timeout of 0 to poll fence status, or VIV_WAIT_INDEFINITE to wait forever.
+ * @return VIV_STATUS_OK if fence finished
+ *         VIV_STATUS_TIMEOUT if timeout expired first
+ *         other if an error occured
+ */
+int viv_fence_finish(struct viv_conn *conn, uint32_t fence, uint32_t timeout);
+
 /** Convenience macro to probe features from state.xml.h:
  * VIV_FEATURE(chipFeatures, FAST_CLEAR)
  * VIV_FEATURE(chipMinorFeatures1, AUTO_DISABLE)
diff --git a/src/fb_old/cube_companion.c b/src/fb_old/cube_companion.c
index 4cbbaff..34c764d 100644
--- a/src/fb_old/cube_companion.c
+++ b/src/fb_old/cube_companion.c
@@ -611,7 +611,7 @@ int main(int argc, char **argv)
 #endif
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 #ifdef EXTRA_DELAYS
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 #endif
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
         etna_set_state(ctx, VIVS_RS_CONFIG,
@@ -632,7 +632,7 @@ int main(int argc, char **argv)
                 VIVS_RS_WINDOW_SIZE_WIDTH(padded_width));
         etna_set_state(ctx, VIVS_RS_KICKER, 0xbeebbeeb);
 #ifdef EXTRA_DELAYS
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_warm_up_rs(ctx, aux_rt->address, aux_rt_ts->address);
 #endif
diff --git a/src/fb_old/etna_test.c b/src/fb_old/etna_test.c
index 8904b6f..5abef5a 100644
--- a/src/fb_old/etna_test.c
+++ b/src/fb_old/etna_test.c
@@ -432,7 +432,7 @@ int main(int argc, char **argv)
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 
         /* Submit first command buffer */
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
@@ -456,7 +456,7 @@ int main(int argc, char **argv)
         etna_set_state(ctx, VIVS_RS_KICKER, 0xbeebbeeb);
 
         /* Submit second command buffer */
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_warm_up_rs(ctx, aux_rt->address, aux_rt_ts->address);
 
diff --git a/src/fb_old/mip_cube_raw.c b/src/fb_old/mip_cube_raw.c
index 1d8e63e..0b10348 100644
--- a/src/fb_old/mip_cube_raw.c
+++ b/src/fb_old/mip_cube_raw.c
@@ -637,7 +637,7 @@ int main(int argc, char **argv)
                 VIVS_RS_WINDOW_SIZE_WIDTH(width));
         etna_set_state(ctx, VIVS_RS_KICKER, 0xbeebbeeb);
 
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
         etna_bswap_queue_swap(buffers);
     }
 #ifdef DUMP
diff --git a/src/fb_old/rotate_cube.c b/src/fb_old/rotate_cube.c
index a0b2777..7340764 100644
--- a/src/fb_old/rotate_cube.c
+++ b/src/fb_old/rotate_cube.c
@@ -566,7 +566,7 @@ int main(int argc, char **argv)
         //exit(1);
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
         etna_set_state(ctx, VIVS_RS_CONFIG,
@@ -588,7 +588,7 @@ int main(int argc, char **argv)
         etna_set_state(ctx, VIVS_RS_KICKER, 0xbeebbeeb);
 
         /* Submit second command buffer */
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_warm_up_rs(ctx, aux_rt->address, aux_rt_ts->address);
 
diff --git a/src/lib/etna_bswap.c b/src/lib/etna_bswap.c
index 9a19527..9c9dd7e 100644
--- a/src/lib/etna_bswap.c
+++ b/src/lib/etna_bswap.c
@@ -144,7 +144,7 @@ int etna_bswap_wait_available(struct etna_bswap_buffers *bufs)
     pthread_mutex_lock(&buf->available_mutex);
     if(!buf->is_available) /* if we're going to wait anyway, flush so that GPU is not idle */
     {
-        etna_flush(bufs->ctx);
+        etna_flush(bufs->ctx, NULL);
     }
     while(!buf->is_available)
     {
@@ -166,7 +166,7 @@ int etna_bswap_queue_swap(struct etna_bswap_buffers *bufs)
 #endif
         return ETNA_INTERNAL_ERROR;
     }
-    if((rv=etna_flush(bufs->ctx)) != ETNA_OK)
+    if((rv=etna_flush(bufs->ctx, NULL)) != ETNA_OK)
         return rv;
     bufs->backbuffer = (bufs->backbuffer + 1) % bufs->num_buffers;
     return ETNA_OK;
diff --git a/src/replay/cube_etna2.c b/src/replay/cube_etna2.c
index 3808db1..9ce91af 100644
--- a/src/replay/cube_etna2.c
+++ b/src/replay/cube_etna2.c
@@ -549,7 +549,7 @@ int main(int argc, char **argv)
         }
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
         etna_set_state(ctx, VIVS_RS_CONFIG,
@@ -571,7 +571,7 @@ int main(int argc, char **argv)
         etna_set_state(ctx, VIVS_RS_KICKER, 0xbeebbeeb);
 
         /* Submit second command buffer */
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_warm_up_rs(ctx, aux_rt->address, aux_rt_ts->address);
 
diff --git a/src/replay/cube_etna2_gc2000.c b/src/replay/cube_etna2_gc2000.c
index 240793d..17ad316 100644
--- a/src/replay/cube_etna2_gc2000.c
+++ b/src/replay/cube_etna2_gc2000.c
@@ -565,7 +565,7 @@ int main(int argc, char **argv)
 
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
         etna_set_state(ctx, VIVS_RS_CONFIG,
@@ -586,7 +586,7 @@ int main(int argc, char **argv)
         etna_set_state(ctx, VIVS_RS_KICKER, 0xbadabeeb);
 
         /* Submit second command buffer */
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_set_state(ctx, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR);
diff --git a/src/replay/etna_test.c b/src/replay/etna_test.c
index 9278b97..de112f0 100644
--- a/src/replay/etna_test.c
+++ b/src/replay/etna_test.c
@@ -401,7 +401,7 @@ int main(int argc, char **argv)
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 
         /* Submit first command buffer */
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
         etna_set_state(ctx, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
@@ -425,7 +425,7 @@ int main(int argc, char **argv)
         etna_set_state(ctx, VIVS_RS_KICKER, 0xbeebbeeb);
 
         /* Submit second command buffer */
-        etna_flush(ctx);
+        etna_flush(ctx, NULL);
 
         etna_warm_up_rs(ctx, aux_rt->address, aux_rt_ts->address);