drm/nouveau/fifo: move PBDMA init to runq

- bumps pbdma timeout to value RM uses on newer HW
- bumps fb timeout to max from boot default
- one/both of these greatly improves stability on // piglit runs

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Reviewed-by: Lyude Paul <lyude@redhat.com>

15 files changed, 36 insertions, 74 deletions

diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index a6cec417012f..e51fb10a04f7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -192,6 +192,9 @@ nvkm_fifo_init(struct nvkm_engine *engine)
 			mask |= BIT(runq->id);
 
 		fifo->func->init_pbdmas(fifo, mask);
+
+		nvkm_runq_foreach(runq, fifo)
+			runq->func->init(runq);
 	}
 
 	fifo->func->init(fifo);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index 3c2f6ce62286..7deb88ec8a9f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -97,8 +97,19 @@ gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit)
 	nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat);
 }
 
+void
+gf100_runq_init(struct nvkm_runq *runq)
+{
+	struct nvkm_device *device = runq->fifo->engine.subdev.device;
+
+	nvkm_mask(device, 0x04013c + (runq->id * 0x2000), 0x10000100, 0x00000000);
+	nvkm_wr32(device, 0x040108 + (runq->id * 0x2000), 0xffffffff); /* INTR */
+	nvkm_wr32(device, 0x04010c + (runq->id * 0x2000), 0xfffffeff); /* INTREN */
+}
+
 static const struct nvkm_runq_func
 gf100_runq = {
+	.init = gf100_runq_init,
 };
 
 void
@@ -653,14 +664,6 @@ gf100_fifo_init(struct nvkm_fifo *base)
 {
 	struct gf100_fifo *fifo = gf100_fifo(base);
 	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	int i;
-
-	/* PBDMA[n] */
-	for (i = 0; i < fifo->pbdma_nr; i++) {
-		nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
-		nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
-		nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
-	}
 
 	nvkm_mask(device, 0x002200, 0x00000001, 0x00000001);
 	nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12);
@@ -716,8 +719,6 @@ gf100_fifo_oneinit(struct nvkm_fifo *base)
 	struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
 	int ret;
 
-	fifo->pbdma_nr = fifo->base.func->runq_nr(&fifo->base);
-
 	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000,
 			      false, &fifo->runlist.mem[0]);
 	if (ret)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h
index b8642490eb2f..82b8467545a8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h
@@ -17,8 +17,6 @@ struct gf100_fifo {
 		u64 mask;
 	} recover;
 
-	int pbdma_nr;
-
 	struct {
 		struct nvkm_memory *mem[2];
 		int active;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index 07fd5b938184..8b6bc79843c7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -164,6 +164,17 @@ static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = {
 	{}
 };
 
+void
+gk104_runq_init(struct nvkm_runq *runq)
+{
+	struct nvkm_device *device = runq->fifo->engine.subdev.device;
+
+	gf100_runq_init(runq);
+
+	nvkm_wr32(device, 0x040148 + (runq->id * 0x2000), 0xffffffff); /* HCE.INTR */
+	nvkm_wr32(device, 0x04014c + (runq->id * 0x2000), 0xffffffff); /* HCE.INTREN */
+}
+
 static u32
 gk104_runq_runm(struct nvkm_runq *runq)
 {
@@ -172,6 +183,7 @@ gk104_runq_runm(struct nvkm_runq *runq)
 
 const struct nvkm_runq_func
 gk104_runq = {
+	.init = gk104_runq_init,
 };
 
 void
@@ -278,10 +290,6 @@ static const struct nvkm_runl_func
 gk104_runl = {
 };
 
-const struct gk104_fifo_pbdma_func
-gk104_fifo_pbdma = {
-};
-
 int
 gk104_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine)
 {
@@ -968,26 +976,9 @@ gk104_fifo_init(struct nvkm_fifo *base)
 {
 	struct gk104_fifo *fifo = gk104_fifo(base);
 	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	int i;
-
-	/* PBDMA[n] */
-	for (i = 0; i < fifo->pbdma_nr; i++) {
-		nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
-		nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
-		nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
-	}
-
-	/* PBDMA[n].HCE */
-	for (i = 0; i < fifo->pbdma_nr; i++) {
-		nvkm_wr32(device, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */
-		nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */
-	}
 
 	nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12);
 
-	if (fifo->func->pbdma->init_timeout)
-		fifo->func->pbdma->init_timeout(fifo);
-
 	nvkm_wr32(device, 0x002100, 0xffffffff);
 	nvkm_wr32(device, 0x002140, 0x7fffffff);
 }
@@ -1054,8 +1045,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
 	struct nvkm_top_device *tdev;
 	int ret, i, j;
 
-	fifo->pbdma_nr = fifo->func->runq_nr(&fifo->base);
-
 	/* Determine runlist configuration from topology device info. */
 	list_for_each_entry(tdev, &device->top->device, head) {
 		const int engn = tdev->engine;
@@ -1158,7 +1147,6 @@ gk104_fifo = {
 	.engine_id = gk104_fifo_engine_id,
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk104_fifo_runlist,
-	.pbdma = &gk104_fifo_pbdma,
 	.nonstall = &gf100_fifo_nonstall,
 	.runl = &gk104_runl,
 	.runq = &gk104_runq,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
index 953177b90e9b..99ae200b2f89 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
@@ -21,8 +21,6 @@ struct gk104_fifo {
 		u32 runm;
 	} recover;
 
-	int pbdma_nr;
-
 	struct {
 		struct nvkm_engine *engine;
 		int runl;
@@ -74,7 +72,6 @@ int gk104_fifo_oneinit(struct nvkm_fifo *);
 void gk104_fifo_init(struct nvkm_fifo *base);
 void gk104_fifo_fini(struct nvkm_fifo *base);
 
-extern const struct gk104_fifo_pbdma_func gk104_fifo_pbdma;
 extern const struct nvkm_enum gk104_fifo_fault_access[];
 extern const struct nvkm_enum gk104_fifo_fault_engine[];
 extern const struct nvkm_enum gk104_fifo_fault_reason[];
@@ -90,14 +87,11 @@ extern const struct gk104_fifo_runlist_func gk110_fifo_runlist;
 void gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *,
 			     struct nvkm_memory *, u32);
 
-extern const struct gk104_fifo_pbdma_func gk208_fifo_pbdma;
 void gk208_fifo_pbdma_init_timeout(struct gk104_fifo *);
 
 extern const struct nvkm_enum gm107_fifo_fault_engine[];
 extern const struct gk104_fifo_runlist_func gm107_fifo_runlist;
 
-extern const struct gk104_fifo_pbdma_func gm200_fifo_pbdma;
-
 extern const struct nvkm_enum gp100_fifo_fault_engine[];
 
 extern const struct nvkm_enum gv100_fifo_fault_access[];
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
index 85705008e0f2..4143a07e3808 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
@@ -96,7 +96,6 @@ gk110_fifo = {
 	.engine_id = gk104_fifo_engine_id,
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
-	.pbdma = &gk104_fifo_pbdma,
 	.nonstall = &gf100_fifo_nonstall,
 	.runl = &gk110_runl,
 	.runq = &gk104_runq,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
index 5224723ce424..0b48fb1b3da4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
@@ -28,22 +28,16 @@
 #include <nvif/class.h>
 
 void
-gk208_fifo_pbdma_init_timeout(struct gk104_fifo *fifo)
+gk208_runq_init(struct nvkm_runq *runq)
 {
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	int i;
+	gk104_runq_init(runq);
 
-	for (i = 0; i < fifo->pbdma_nr; i++)
-		nvkm_wr32(device, 0x04012c + (i * 0x2000), 0x0000ffff);
+	nvkm_wr32(runq->fifo->engine.subdev.device, 0x04012c + (runq->id * 0x2000), 0x000f4240);
 }
 
-const struct gk104_fifo_pbdma_func
-gk208_fifo_pbdma = {
-	.init_timeout = gk208_fifo_pbdma_init_timeout,
-};
-
 const struct nvkm_runq_func
 gk208_runq = {
+	.init = gk208_runq_init,
 };
 
 static int
@@ -75,7 +69,6 @@ gk208_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.nonstall = &gf100_fifo_nonstall,
-	.pbdma = &gk208_fifo_pbdma,
 	.runl = &gk110_runl,
 	.runq = &gk208_runq,
 	.engn = &gk104_engn,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
index 8ce3bb39f651..3d3db802b824 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
@@ -46,7 +46,6 @@ gk20a_fifo = {
 	.engine_id = gk104_fifo_engine_id,
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
-	.pbdma = &gk208_fifo_pbdma,
 	.nonstall = &gf100_fifo_nonstall,
 	.runl = &gk110_runl,
 	.runq = &gk208_runq,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
index 9a608c02c668..a85f30768b74 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
@@ -136,7 +136,6 @@ gm107_fifo = {
 	.engine_id = gk104_fifo_engine_id,
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
-	.pbdma = &gk208_fifo_pbdma,
 	.nonstall = &gf100_fifo_nonstall,
 	.runl = &gm107_runl,
 	.runq = &gk208_runq,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
index 1ecfbea3680a..131e4ae61590 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
@@ -32,11 +32,6 @@ gm200_fifo_runq_nr(struct nvkm_fifo *fifo)
 	return nvkm_rd32(fifo->engine.subdev.device, 0x002004) & 0x000000ff;
 }
 
-const struct gk104_fifo_pbdma_func
-gm200_fifo_pbdma = {
-	.init_timeout = gk208_fifo_pbdma_init_timeout,
-};
-
 int
 gm200_fifo_chid_nr(struct nvkm_fifo *fifo)
 {
@@ -65,7 +60,6 @@ gm200_fifo = {
 	.engine_id = gk104_fifo_engine_id,
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
-	.pbdma = &gm200_fifo_pbdma,
 	.nonstall = &gf100_fifo_nonstall,
 	.runl = &gm107_runl,
 	.runq = &gk208_runq,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
index 1e9c5ea9cd13..b05ede6dd392 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
@@ -108,7 +108,6 @@ gp100_fifo = {
 	.engine_id = gk104_fifo_engine_id,
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
-	.pbdma = &gm200_fifo_pbdma,
 	.nonstall = &gf100_fifo_nonstall,
 	.runl = &gp100_runl,
 	.runq = &gk208_runq,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
index cc9ddc65e881..251d5e5826db 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
@@ -45,6 +45,7 @@ gv100_engn_ce = {
 
 const struct nvkm_runq_func
 gv100_runq = {
+	.init = gk208_runq_init,
 };
 
 void
@@ -335,7 +336,6 @@ gv100_fifo = {
 	.engine_id = gk104_fifo_engine_id,
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gv100_fifo_runlist,
-	.pbdma = &gm200_fifo_pbdma,
 	.nonstall = &gf100_fifo_nonstall,
 	.runl = &gv100_runl,
 	.runq = &gv100_runq,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
index 8657e5c84c64..c30d7ac07803 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
@@ -5,6 +5,7 @@
 #include <engine/fifo.h>
 struct nvkm_cgrp;
 struct nvkm_memory;
+struct nvkm_runq;
 struct gk104_fifo;
 struct gk104_fifo_chan;
 
@@ -59,10 +60,6 @@ struct nvkm_fifo_func {
 			       struct nvkm_memory *, int entries);
 	} *runlist;
 
-	const struct gk104_fifo_pbdma_func {
-		void (*init_timeout)(struct gk104_fifo *);
-	} *pbdma;
-
 	const struct nvkm_event_func *nonstall;
 
 	const struct nvkm_runl_func *runl;
@@ -118,6 +115,7 @@ int gf100_fifo_chid_ctor(struct nvkm_fifo *, int);
 int gf100_fifo_runq_nr(struct nvkm_fifo *);
 void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
 extern const struct nvkm_event_func gf100_fifo_nonstall;
+void gf100_runq_init(struct nvkm_runq *);
 extern const struct nvkm_engn_func gf100_engn_sw;
 
 int gk104_fifo_chid_nr(struct nvkm_fifo *);
@@ -131,6 +129,7 @@ void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *);
 void gk104_fifo_recover_chan(struct nvkm_fifo *, int);
 int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *);
 extern const struct nvkm_runq_func gk104_runq;
+void gk104_runq_init(struct nvkm_runq *);
 extern const struct nvkm_engn_func gk104_engn;
 extern const struct nvkm_engn_func gk104_engn_ce;
 
@@ -140,6 +139,7 @@ extern const struct nvkm_cgrp_func gk110_cgrp;
 extern const struct nvkm_chan_func gk110_chan;
 
 extern const struct nvkm_runq_func gk208_runq;
+void gk208_runq_init(struct nvkm_runq *);
 
 void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
 extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h
index 3089d9a8e70e..e4909161c9bc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h
@@ -5,6 +5,7 @@
 
 struct nvkm_runq {
 	const struct nvkm_runq_func {
+		void (*init)(struct nvkm_runq *);
 	} *func;
 	struct nvkm_fifo *fifo;
 	int id;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
index aa933231ec60..5fcc4ebb9f5b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
@@ -94,11 +94,6 @@ tu102_fifo_fault_engine[] = {
 	{}
 };
 
-static const struct gk104_fifo_pbdma_func
-tu102_fifo_pbdma = {
-	.init_timeout = gk208_fifo_pbdma_init_timeout,
-};
-
 static void
 tu102_fifo_recover_work(struct work_struct *w)
 {
@@ -463,7 +458,6 @@ tu102_fifo = {
 	.engine_id = gk104_fifo_engine_id,
 	.recover_chan = tu102_fifo_recover_chan,
 	.runlist = &tu102_fifo_runlist,
-	.pbdma = &tu102_fifo_pbdma,
 	.nonstall = &gf100_fifo_nonstall,
 	.runl = &tu102_runl,
 	.runq = &gv100_runq,