nouveau: pushbuf code, now with 50% less suck!

Far more efficient, if not a bit more complicated.  Hopefully not too
buggy still.

This commit will potentially expose some unrelated bugs, fixes for them
will follow "real soon now".

5 files changed, 155 insertions, 123 deletions

diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.c b/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.c
index 3bb7c49762..f8a8ba04f6 100644
--- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.c
+++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.c
@@ -28,23 +28,23 @@
 #include "nouveau_dma.h"
 #include "nouveau_local.h"
 
-static __inline__ uint32_t
+static inline uint32_t
 READ_GET(struct nouveau_channel_priv *nvchan)
 {
-	return ((*nvchan->get - nvchan->dma.base) >> 2);
+	return *nvchan->get;
 }
 
-static __inline__ void
+static inline void
 WRITE_PUT(struct nouveau_channel_priv *nvchan, uint32_t val)
 {
-	uint32_t put = ((val << 2) + nvchan->dma.base);
+	uint32_t put = ((val << 2) + nvchan->dma->base);
 	volatile int dum;
 
 	NOUVEAU_DMA_BARRIER;
 	dum = READ_GET(nvchan);
 
 	*nvchan->put = put;
-	nvchan->dma.put = val;
+	nvchan->dma->put = val;
 #ifdef NOUVEAU_DMA_TRACE
 	NOUVEAU_MSG("WRITE_PUT %d/0x%08x\n", nvchan->drm.channel, put);
 #endif
@@ -52,16 +52,30 @@ WRITE_PUT(struct nouveau_channel_priv *nvchan, uint32_t val)
 	NOUVEAU_DMA_BARRIER;
 }
 
+static inline int
+LOCAL_GET(struct nouveau_dma_priv *dma, uint32_t *val)
+{
+	uint32_t get = *val;
+
+	if (get >= dma->base && get <= (dma->base + (dma->max << 2))) {
+		*val = (get - dma->base) >> 2;
+		return 1;
+	}
+
+	return 0;
+}
+
 void
 nouveau_dma_channel_init(struct nouveau_channel *chan)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
 	int i;
 
-	nvchan->dma.base = nvchan->drm.put_base;
-	nvchan->dma.cur  = nvchan->dma.put = 0;
-	nvchan->dma.max  = (nvchan->drm.cmdbuf_size >> 2) - 2;
-	nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur;
+	nvchan->dma = &nvchan->dma_master;
+	nvchan->dma->base = nvchan->drm.put_base;
+	nvchan->dma->cur  = nvchan->dma->put = 0;
+	nvchan->dma->max  = (nvchan->drm.cmdbuf_size >> 2) - 2;
+	nvchan->dma->free = nvchan->dma->max - nvchan->dma->cur;
 
 	RING_SPACE_CH(chan, RING_SKIPS);
 	for (i = 0; i < RING_SKIPS; i++)
@@ -73,54 +87,51 @@ nouveau_dma_channel_init(struct nouveau_channel *chan)
 		return - EBUSY;                                                \
 } while(0)
 
-#define IN_MASTER_RING(chan, ptr) ((ptr) <= (chan)->dma.max)
-
 int
 nouveau_dma_wait(struct nouveau_channel *chan, int size)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+	struct nouveau_dma_priv *dma = nvchan->dma;
 	uint32_t get, t_start;
 
 	FIRE_RING_CH(chan);
 
 	t_start = NOUVEAU_TIME_MSEC();
-	while (nvchan->dma.free < size) {
+	while (dma->free < size) {
 		CHECK_TIMEOUT();
 
 		get = READ_GET(nvchan);
-		if (!IN_MASTER_RING(nvchan, get))
+		if (!LOCAL_GET(dma, &get))
 			continue;
 
-		if (nvchan->dma.put >= get) {
-			nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur;
+		if (dma->put >= get) {
+			dma->free = dma->max - dma->cur;
 
-			if (nvchan->dma.free < size) {
+			if (dma->free < size) {
 #ifdef NOUVEAU_DMA_DEBUG
-				nvchan->dma.push_free = 1;
+				dma->push_free = 1;
 #endif
-				OUT_RING_CH(chan,
-					    0x20000000 | nvchan->dma.base);
+				OUT_RING_CH(chan, 0x20000000 | dma->base);
 				if (get <= RING_SKIPS) {
 					/*corner case - will be idle*/
-					if (nvchan->dma.put <= RING_SKIPS)
+					if (dma->put <= RING_SKIPS)
 						WRITE_PUT(nvchan,
 							  RING_SKIPS + 1);
 
 					do {
 						CHECK_TIMEOUT();
 						get = READ_GET(nvchan);
-						if (!IN_MASTER_RING(nvchan,
-								    get))
-							continue;
+						if (!LOCAL_GET(dma, &get))
+							get = 0;
 					} while (get <= RING_SKIPS);
 				}
 
 				WRITE_PUT(nvchan, RING_SKIPS);
-				nvchan->dma.cur  = nvchan->dma.put = RING_SKIPS;
-				nvchan->dma.free = get - (RING_SKIPS + 1);
+				dma->cur  = dma->put = RING_SKIPS;
+				dma->free = get - (RING_SKIPS + 1);
 			}
 		} else {
-			nvchan->dma.free = get - nvchan->dma.cur - 1;
+			dma->free = get - dma->cur - 1;
 		}
 	}
 
@@ -135,7 +146,7 @@ nouveau_dma_parse_pushbuf(struct nouveau_channel *chan, int get, int put)
 	unsigned mthd_count = 0;
 	
 	while (get != put) {
-		uint32_t gpuget = (get << 2) + nvchan->dma.base;
+		uint32_t gpuget = (get << 2) + nvchan->drm.put_base;
 		uint32_t data;
 
 		if (get < 0 || get >= nvchan->drm.cmdbuf_size) {
@@ -188,21 +199,21 @@ void
 nouveau_dma_kickoff(struct nouveau_channel *chan)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+	struct nouveau_dma_priv *dma = nvchan->dma;
 
-	if (nvchan->dma.cur == nvchan->dma.put)
+	if (dma->cur == dma->put)
 		return;
 
 #ifdef NOUVEAU_DMA_DEBUG
-	if (nvchan->dma.push_free) {
-		NOUVEAU_ERR("Packet incomplete: %d left\n",
-			    nvchan->dma.push_free);
+	if (dma->push_free) {
+		NOUVEAU_ERR("Packet incomplete: %d left\n", dma->push_free);
 		return;
 	}
 #endif
 
 #ifdef NOUVEAU_DMA_DUMP_POSTRELOC_PUSHBUF
-	nouveau_dma_parse_pushbuf(chan, nvchan->dma.put, nvchan->dma.cur);
+	nouveau_dma_parse_pushbuf(chan, dma->put, dma->cur);
 #endif
 
-	WRITE_PUT(nvchan, nvchan->dma.cur);
+	WRITE_PUT(nvchan, dma->cur);
 }
diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.h b/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.h
index 940a196bfe..cfa6d26e82 100644
--- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.h
+++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.h
@@ -42,34 +42,37 @@ static inline void
 nouveau_dma_out(struct nouveau_channel *chan, uint32_t data)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+	struct nouveau_dma_priv *dma = nvchan->dma;
 
 #ifdef NOUVEAU_DMA_DEBUG
-	if (nvchan->dma.push_free == 0) {
-		NOUVEAU_ERR("No space left in packet. Error at %s\n",faulty);
+	if (dma->push_free == 0) {
+		NOUVEAU_ERR("No space left in packet at %s\n", faulty);
 		return;
 	}
-	nvchan->dma.push_free--;
+	dma->push_free--;
 #endif
 #ifdef NOUVEAU_DMA_TRACE
 	{
-		uint32_t offset = (nvchan->dma.cur << 2) + nvchan->dma.base;
+		uint32_t offset = (dma->cur << 2) + dma->base;
 		NOUVEAU_MSG("\tOUT_RING %d/0x%08x -> 0x%08x\n",
 			    nvchan->drm.channel, offset, data);
 	}
 #endif
-	nvchan->pushbuf[nvchan->dma.cur++] = data;
+	nvchan->pushbuf[dma->cur + (dma->base - nvchan->drm.put_base)/4] = data;
+	dma->cur++;
 }
 
 static inline void
 nouveau_dma_outp(struct nouveau_channel *chan, uint32_t *ptr, int size)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
-	(void)chan;
+	struct nouveau_dma_priv *dma = nvchan->dma;
+	(void)dma;
 
 #ifdef NOUVEAU_DMA_DEBUG
-	if (nvchan->dma.push_free < size) {
+	if (dma->push_free < size) {
 		NOUVEAU_ERR("Packet too small.  Free=%d, Need=%d\n",
-			    nvchan->dma.push_free, size);
+			    dma->push_free, size);
 		return;
 	}
 #endif
@@ -79,11 +82,11 @@ nouveau_dma_outp(struct nouveau_channel *chan, uint32_t *ptr, int size)
 		ptr++;
 	}
 #else
-	memcpy(&nvchan->pushbuf[nvchan->dma.cur], ptr, size << 2);
+	memcpy(&nvchan->pushbuf[dma->cur], ptr, size << 2);
 #ifdef NOUVEAU_DMA_DEBUG
-	nvchan->dma.push_free -= size;
+	dma->push_free -= size;
 #endif
-	nvchan->dma.cur += size;
+	dma->cur += size;
 #endif
 }
 
@@ -91,14 +94,15 @@ static inline void
 nouveau_dma_space(struct nouveau_channel *chan, int size)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+	struct nouveau_dma_priv *dma = nvchan->dma;
 
-	if (nvchan->dma.free < size) {
+	if (dma->free < size) {
 		if (nouveau_dma_wait(chan, size) && chan->hang_notify)
 			chan->hang_notify(chan);
 	}
-	nvchan->dma.free -= size;
+	dma->free -= size;
 #ifdef NOUVEAU_DMA_DEBUG
-	nvchan->dma.push_free = size;
+	dma->push_free = size;
 #endif
 }
 
@@ -107,7 +111,8 @@ nouveau_dma_begin(struct nouveau_channel *chan, struct nouveau_grobj *grobj,
 		  int method, int size, const char* file, int line)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
-	(void)nvchan;
+	struct nouveau_dma_priv *dma = nvchan->dma;
+	(void)dma;
 
 #ifdef NOUVEAU_DMA_TRACE
 	NOUVEAU_MSG("BEGIN_RING %d/%08x/%d/0x%04x/%d\n", nvchan->drm.channel,
@@ -115,9 +120,9 @@ nouveau_dma_begin(struct nouveau_channel *chan, struct nouveau_grobj *grobj,
 #endif
 
 #ifdef NOUVEAU_DMA_DEBUG
-	if (nvchan->dma.push_free) {
-		NOUVEAU_ERR("Previous packet incomplete: %d left. Error at %s\n",
-			    nvchan->dma.push_free,faulty);
+	if (dma->push_free) {
+		NOUVEAU_ERR("Previous packet incomplete: %d left at %s\n",
+			    dma->push_free, faulty);
 		return;
 	}
 	sprintf(faulty,"%s:%d",file,line);
diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_drmif.h b/src/mesa/drivers/dri/nouveau_winsys/nouveau_drmif.h
index 11a15d632c..724677961e 100644
--- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_drmif.h
+++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_drmif.h
@@ -128,10 +128,10 @@ struct nouveau_pushbuf_bo {
 
 struct nouveau_pushbuf_priv {
 	struct nouveau_pushbuf base;
-	struct nouveau_pushbuf *next;
 
-	struct nouveau_resource *res;
-	struct nouveau_fence *fence;
+	unsigned nop_jump;
+	unsigned start;
+	unsigned size;
 
 	uint64_t buffers;
 	int nr_buffers;
@@ -149,13 +149,23 @@ extern int
 nouveau_pushbuf_init(struct nouveau_channel *);
 
 extern int
-nouveau_pushbuf_flush(struct nouveau_channel *);
+nouveau_pushbuf_flush(struct nouveau_channel *, unsigned min);
 
 extern int
 nouveau_pushbuf_emit_reloc(struct nouveau_channel *, void *ptr,
 			   struct nouveau_bo *, uint32_t data, uint32_t flags,
 			   uint32_t vor, uint32_t tor);
 
+struct nouveau_dma_priv {
+	uint32_t base;
+	uint32_t max;
+	uint32_t cur;
+	uint32_t put;
+	uint32_t free;
+
+	int push_free;
+} dma;
+
 struct nouveau_channel_priv {
 	struct nouveau_channel base;
 
@@ -169,21 +179,15 @@ struct nouveau_channel_priv {
 	volatile uint32_t *get;
 	volatile uint32_t *ref_cnt;
 
-	struct {
-		uint32_t base, max;
-		uint32_t cur, put;
-		uint32_t free;
-
-		int push_free;
-	} dma;
+	struct nouveau_dma_priv dma_master;
+	struct nouveau_dma_priv dma_bufmgr;
+	struct nouveau_dma_priv *dma;
 
 	struct nouveau_fence *fence_head;
 	struct nouveau_fence *fence_tail;
 	uint32_t fence_sequence;
 
-	struct nouveau_resource *pb_heap;
-	struct nouveau_pushbuf *pb_head;
-	struct nouveau_pushbuf *pb_tail;
+	struct nouveau_pushbuf_priv pb;
 
 	unsigned user_charge;
 };
diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_pushbuf.c b/src/mesa/drivers/dri/nouveau_winsys/nouveau_pushbuf.c
index 1175441789..148406b85a 100644
--- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_pushbuf.c
+++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_pushbuf.c
@@ -27,28 +27,34 @@
 #include "nouveau_drmif.h"
 #include "nouveau_dma.h"
 
-#define PB_RSVD_DWORDS 2
+#define PB_BUFMGR_DWORDS   (4096 / 2)
+#define PB_MIN_USER_DWORDS  2048
 
 static int
-nouveau_pushbuf_space(struct nouveau_channel *chan)
+nouveau_pushbuf_space(struct nouveau_channel *chan, unsigned min)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
-	struct nouveau_pushbuf_priv *nvpb;
+	struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
 
-	nvpb = calloc(1, sizeof(struct nouveau_pushbuf_priv));
-	if (!nvpb)
-		return -ENOMEM;
+	assert((min + 1) <= nvchan->dma->max);
 
-	while (nouveau_resource_alloc(nvchan->pb_heap, 0x2100, NULL,
-				      &nvpb->res)) {
-		nouveau_fence_flush(chan);
-	}
+	/* Wait for enough space in push buffer */
+	min = min < PB_MIN_USER_DWORDS ? PB_MIN_USER_DWORDS : min;
+	min += 1; /* a bit extra for the NOP */
+	if (nvchan->dma->free < min)
+		WAIT_RING_CH(chan, min);
+
+	/* Insert NOP, may turn into a jump later */
+	RING_SPACE_CH(chan, 1);
+	nvpb->nop_jump = nvchan->dma->cur;
+	OUT_RING_CH(chan, 0);
 
+	/* Any remaining space is available to the user */
+	nvpb->start = nvchan->dma->cur;
+	nvpb->size = nvchan->dma->free;
 	nvpb->base.channel = chan;
-	nvpb->base.remaining = (nvpb->res->size / 4) - PB_RSVD_DWORDS;
-	nvpb->base.cur = &nvchan->pushbuf[nvpb->res->start/4];
-	nvchan->pb_tail = &nvpb->base;
-	nvchan->base.pushbuf = nvchan->pb_tail;
+	nvpb->base.remaining = nvpb->size;
+	nvpb->base.cur = &nvchan->pushbuf[nvpb->start];
 
 	return 0;
 }
@@ -57,53 +63,65 @@ int
 nouveau_pushbuf_init(struct nouveau_channel *chan)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+	struct nouveau_dma_priv *m = &nvchan->dma_master;
+	struct nouveau_dma_priv *b = &nvchan->dma_bufmgr;
+	int i;
 
 	if (!nvchan)
 		return -EINVAL;
 
-	/* Everything except first 4KiB of the push buffer is managed by us */
-	if (nouveau_resource_init(&nvchan->pb_heap, 4096,
-				  nvchan->drm.cmdbuf_size - 4096))
-		return -EINVAL;
-
-	/* Shrink master ring to 4KiB */
-	assert(nvchan->dma.cur <= (4096/4));
-	nvchan->dma.max = (4096 / 4) - 2;
-	nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur;
-
-	assert(!nouveau_pushbuf_space(chan));
+	/* Reassign last bit of push buffer for a "separate" bufmgr
+	 * ring buffer
+	 */
+	m->max -= PB_BUFMGR_DWORDS;
+	m->free -= PB_BUFMGR_DWORDS;
+
+	b->base = m->base + ((m->max + 2) << 2);
+	b->max = PB_BUFMGR_DWORDS - 2;
+	b->cur = b->put = 0;
+	b->free = b->max - b->cur;
+
+	/* Some NOPs just to be safe
+	 *XXX: RING_SKIPS
+	 */
+	nvchan->dma = b;
+	RING_SPACE_CH(chan, 8);
+	for (i = 0; i < 8; i++)
+		OUT_RING_CH(chan, 0);
+	nvchan->dma = m;
+
+	nouveau_pushbuf_space(chan, 0);
+	chan->pushbuf = &nvchan->pb.base;
 
 	return 0;
 }
 
-static void
-nouveau_pushbuf_fence_signalled(void *priv)
-{
-	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(priv);
-
-	nouveau_fence_del(&nvpb->fence);
-	nouveau_resource_free(&nvpb->res);
-	free(nvpb);
-}
-
 /* This would be our TTM "superioctl" */
 int
-nouveau_pushbuf_flush(struct nouveau_channel *chan)
+nouveau_pushbuf_flush(struct nouveau_channel *chan, unsigned min)
 {
 	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
-	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf);
+	struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
 	struct nouveau_pushbuf_bo *pbbo;
 	struct nouveau_fence *fence = NULL;
 	int ret;
 
-	if (nvpb->base.remaining == (nvpb->res->size / 4) - PB_RSVD_DWORDS)
+	if (nvpb->base.remaining == nvpb->size)
 		return 0;
-	nvchan->pb_tail = NULL;
+
+	nvpb->size -= nvpb->base.remaining;
+	nvchan->dma->cur += nvpb->size;
+	nvchan->dma->free -= nvpb->size;
+	assert(nvchan->dma->cur <= nvchan->dma->max);
 
 	ret = nouveau_fence_new(chan, &fence);
 	if (ret)
 		return ret;
 
+	nvchan->dma = &nvchan->dma_bufmgr;
+	nvchan->pushbuf[nvpb->nop_jump] = 0x20000000 |
+		(nvchan->dma->base + (nvchan->dma->cur << 2));
+
 	/* Validate buffers + apply relocations */
 	nvchan->user_charge = 0;
 	while ((pbbo = ptr_to_pbbo(nvpb->buffers))) {
@@ -142,25 +160,19 @@ nouveau_pushbuf_flush(struct nouveau_channel *chan)
 	}
 	nvpb->nr_buffers = 0;
 
-	/* Emit JMP to indirect pushbuf */
+	/* Switch back to user's ring */
 	RING_SPACE_CH(chan, 1);
-	OUT_RING_CH(chan, 0x20000000 | nvpb->res->start);
+	OUT_RING_CH(chan, 0x20000000 | ((nvpb->start << 2) +
+					nvchan->dma_master.base));
+	nvchan->dma = &nvchan->dma_master;
 
-	/* Add JMP back to master pushbuf from indirect pushbuf */
-	(*nvpb->base.cur++) =
-		0x20000000 | ((nvchan->dma.cur << 2) + nvchan->dma.base);
-
-	/* Fence */
-	nvpb->fence = fence;
-	nouveau_fence_signal_cb(nvpb->fence, nouveau_pushbuf_fence_signalled,
-				nvpb);
-	nouveau_fence_emit(nvpb->fence);
-
-	/* Kickoff */
+	/* Fence + kickoff */
+	nouveau_fence_emit(fence);
 	FIRE_RING_CH(chan);
+	nouveau_fence_del(&fence);
 
 	/* Allocate space for next push buffer */
-	assert(!nouveau_pushbuf_space(chan));
+	assert(!nouveau_pushbuf_space(chan, min));
 
 	return 0;
 }
@@ -168,8 +180,7 @@ nouveau_pushbuf_flush(struct nouveau_channel *chan)
 static struct nouveau_pushbuf_bo *
 nouveau_pushbuf_emit_buffer(struct nouveau_channel *chan, struct nouveau_bo *bo)
 {
-	struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
-	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(nvchan->pb_tail);
+	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf);
 	struct nouveau_pushbuf_bo *pbbo = ptr_to_pbbo(nvpb->buffers);
 
 	while (pbbo) {
diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_winsys.c b/src/mesa/drivers/dri/nouveau_winsys/nouveau_winsys.c
index b31e0dcd1d..403647ab17 100644
--- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_winsys.c
+++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_winsys.c
@@ -40,8 +40,9 @@ nouveau_pipe_dma_beginp(struct nouveau_grobj *grobj, int mthd, int size)
 	struct nouveau_channel *chan = grobj->channel;
 	uint32_t *pushbuf;
 
-	if (chan->pushbuf->remaining < (size + 1))
-		nouveau_pushbuf_flush(chan);
+	if (chan->pushbuf->remaining < (size + 1)) {
+		nouveau_pushbuf_flush(chan, size + 1);
+	}
 
 	pushbuf = chan->pushbuf->cur;
 	chan->pushbuf->cur += (size + 1);
@@ -54,7 +55,7 @@ nouveau_pipe_dma_beginp(struct nouveau_grobj *grobj, int mthd, int size)
 void
 nouveau_pipe_dma_kickoff(struct nouveau_channel *chan)
 {
-	nouveau_pushbuf_flush(chan);
+	nouveau_pushbuf_flush(chan, 0);
 }
 
 static int