summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/amd/amdkfd
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd')
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device.c324
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c4
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_migrate.c19
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_migrate.h5
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_priv.h1
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_svm.c31
6 files changed, 250 insertions, 134 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 16a57b70cc1a..064d42acd54e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -32,6 +32,7 @@
#include "amdgpu_amdkfd.h"
#include "kfd_smi_events.h"
#include "kfd_migrate.h"
+#include "amdgpu.h"
#define MQD_SIZE_ALIGNED 768
@@ -52,41 +53,6 @@ extern const struct kfd2kgd_calls aldebaran_kfd2kgd;
extern const struct kfd2kgd_calls gfx_v10_kfd2kgd;
extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd;
-static const struct kfd2kgd_calls *kfd2kgd_funcs[] = {
-#ifdef KFD_SUPPORT_IOMMU_V2
-#ifdef CONFIG_DRM_AMDGPU_CIK
- [CHIP_KAVERI] = &gfx_v7_kfd2kgd,
-#endif
- [CHIP_CARRIZO] = &gfx_v8_kfd2kgd,
- [CHIP_RAVEN] = &gfx_v9_kfd2kgd,
-#endif
-#ifdef CONFIG_DRM_AMDGPU_CIK
- [CHIP_HAWAII] = &gfx_v7_kfd2kgd,
-#endif
- [CHIP_TONGA] = &gfx_v8_kfd2kgd,
- [CHIP_FIJI] = &gfx_v8_kfd2kgd,
- [CHIP_POLARIS10] = &gfx_v8_kfd2kgd,
- [CHIP_POLARIS11] = &gfx_v8_kfd2kgd,
- [CHIP_POLARIS12] = &gfx_v8_kfd2kgd,
- [CHIP_VEGAM] = &gfx_v8_kfd2kgd,
- [CHIP_VEGA10] = &gfx_v9_kfd2kgd,
- [CHIP_VEGA12] = &gfx_v9_kfd2kgd,
- [CHIP_VEGA20] = &gfx_v9_kfd2kgd,
- [CHIP_RENOIR] = &gfx_v9_kfd2kgd,
- [CHIP_ARCTURUS] = &arcturus_kfd2kgd,
- [CHIP_ALDEBARAN] = &aldebaran_kfd2kgd,
- [CHIP_NAVI10] = &gfx_v10_kfd2kgd,
- [CHIP_NAVI12] = &gfx_v10_kfd2kgd,
- [CHIP_NAVI14] = &gfx_v10_kfd2kgd,
- [CHIP_SIENNA_CICHLID] = &gfx_v10_3_kfd2kgd,
- [CHIP_NAVY_FLOUNDER] = &gfx_v10_3_kfd2kgd,
- [CHIP_VANGOGH] = &gfx_v10_3_kfd2kgd,
- [CHIP_DIMGREY_CAVEFISH] = &gfx_v10_3_kfd2kgd,
- [CHIP_BEIGE_GOBY] = &gfx_v10_3_kfd2kgd,
- [CHIP_YELLOW_CARP] = &gfx_v10_3_kfd2kgd,
- [CHIP_CYAN_SKILLFISH] = &gfx_v10_kfd2kgd,
-};
-
#ifdef KFD_SUPPORT_IOMMU_V2
static const struct kfd_device_info kaveri_device_info = {
.asic_family = CHIP_KAVERI,
@@ -468,6 +434,7 @@ static const struct kfd_device_info navi10_device_info = {
.needs_iommu_device = false,
.supports_cwsr = true,
.needs_pci_atomics = true,
+ .no_atomic_fw_version = 145,
.num_sdma_engines = 2,
.num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 8,
@@ -487,6 +454,7 @@ static const struct kfd_device_info navi12_device_info = {
.needs_iommu_device = false,
.supports_cwsr = true,
.needs_pci_atomics = true,
+ .no_atomic_fw_version = 145,
.num_sdma_engines = 2,
.num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 8,
@@ -506,6 +474,7 @@ static const struct kfd_device_info navi14_device_info = {
.needs_iommu_device = false,
.supports_cwsr = true,
.needs_pci_atomics = true,
+ .no_atomic_fw_version = 145,
.num_sdma_engines = 2,
.num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 8,
@@ -525,6 +494,7 @@ static const struct kfd_device_info sienna_cichlid_device_info = {
.needs_iommu_device = false,
.supports_cwsr = true,
.needs_pci_atomics = true,
+ .no_atomic_fw_version = 92,
.num_sdma_engines = 4,
.num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 8,
@@ -544,6 +514,7 @@ static const struct kfd_device_info navy_flounder_device_info = {
.needs_iommu_device = false,
.supports_cwsr = true,
.needs_pci_atomics = true,
+ .no_atomic_fw_version = 92,
.num_sdma_engines = 2,
.num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 8,
@@ -562,7 +533,8 @@ static const struct kfd_device_info vangogh_device_info = {
.mqd_size_aligned = MQD_SIZE_ALIGNED,
.needs_iommu_device = false,
.supports_cwsr = true,
- .needs_pci_atomics = false,
+ .needs_pci_atomics = true,
+ .no_atomic_fw_version = 92,
.num_sdma_engines = 1,
.num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
@@ -582,6 +554,7 @@ static const struct kfd_device_info dimgrey_cavefish_device_info = {
.needs_iommu_device = false,
.supports_cwsr = true,
.needs_pci_atomics = true,
+ .no_atomic_fw_version = 92,
.num_sdma_engines = 2,
.num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 8,
@@ -601,6 +574,7 @@ static const struct kfd_device_info beige_goby_device_info = {
.needs_iommu_device = false,
.supports_cwsr = true,
.needs_pci_atomics = true,
+ .no_atomic_fw_version = 92,
.num_sdma_engines = 1,
.num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 8,
@@ -619,7 +593,8 @@ static const struct kfd_device_info yellow_carp_device_info = {
.mqd_size_aligned = MQD_SIZE_ALIGNED,
.needs_iommu_device = false,
.supports_cwsr = true,
- .needs_pci_atomics = false,
+ .needs_pci_atomics = true,
+ .no_atomic_fw_version = 92,
.num_sdma_engines = 1,
.num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
@@ -644,63 +619,202 @@ static const struct kfd_device_info cyan_skillfish_device_info = {
.num_sdma_queues_per_engine = 8,
};
-/* For each entry, [0] is regular and [1] is virtualisation device. */
-static const struct kfd_device_info *kfd_supported_devices[][2] = {
-#ifdef KFD_SUPPORT_IOMMU_V2
- [CHIP_KAVERI] = {&kaveri_device_info, NULL},
- [CHIP_CARRIZO] = {&carrizo_device_info, NULL},
-#endif
- [CHIP_RAVEN] = {&raven_device_info, NULL},
- [CHIP_HAWAII] = {&hawaii_device_info, NULL},
- [CHIP_TONGA] = {&tonga_device_info, NULL},
- [CHIP_FIJI] = {&fiji_device_info, &fiji_vf_device_info},
- [CHIP_POLARIS10] = {&polaris10_device_info, &polaris10_vf_device_info},
- [CHIP_POLARIS11] = {&polaris11_device_info, NULL},
- [CHIP_POLARIS12] = {&polaris12_device_info, NULL},
- [CHIP_VEGAM] = {&vegam_device_info, NULL},
- [CHIP_VEGA10] = {&vega10_device_info, &vega10_vf_device_info},
- [CHIP_VEGA12] = {&vega12_device_info, NULL},
- [CHIP_VEGA20] = {&vega20_device_info, NULL},
- [CHIP_RENOIR] = {&renoir_device_info, NULL},
- [CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info},
- [CHIP_ALDEBARAN] = {&aldebaran_device_info, &aldebaran_device_info},
- [CHIP_NAVI10] = {&navi10_device_info, NULL},
- [CHIP_NAVI12] = {&navi12_device_info, &navi12_device_info},
- [CHIP_NAVI14] = {&navi14_device_info, NULL},
- [CHIP_SIENNA_CICHLID] = {&sienna_cichlid_device_info, &sienna_cichlid_device_info},
- [CHIP_NAVY_FLOUNDER] = {&navy_flounder_device_info, &navy_flounder_device_info},
- [CHIP_VANGOGH] = {&vangogh_device_info, NULL},
- [CHIP_DIMGREY_CAVEFISH] = {&dimgrey_cavefish_device_info, &dimgrey_cavefish_device_info},
- [CHIP_BEIGE_GOBY] = {&beige_goby_device_info, &beige_goby_device_info},
- [CHIP_YELLOW_CARP] = {&yellow_carp_device_info, NULL},
- [CHIP_CYAN_SKILLFISH] = {&cyan_skillfish_device_info, NULL},
-};
-
static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
unsigned int chunk_size);
static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
static int kfd_resume(struct kfd_dev *kfd);
-struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
- struct pci_dev *pdev, unsigned int asic_type, bool vf)
+struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, bool vf)
{
struct kfd_dev *kfd;
const struct kfd_device_info *device_info;
const struct kfd2kgd_calls *f2g;
+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+ struct pci_dev *pdev = adev->pdev;
- if (asic_type >= sizeof(kfd_supported_devices) / (sizeof(void *) * 2)
- || asic_type >= sizeof(kfd2kgd_funcs) / sizeof(void *)) {
- dev_err(kfd_device, "asic_type %d out of range\n", asic_type);
- return NULL; /* asic_type out of range */
+ switch (adev->asic_type) {
+#ifdef KFD_SUPPORT_IOMMU_V2
+#ifdef CONFIG_DRM_AMDGPU_CIK
+ case CHIP_KAVERI:
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &kaveri_device_info;
+ f2g = &gfx_v7_kfd2kgd;
+ break;
+#endif
+ case CHIP_CARRIZO:
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &carrizo_device_info;
+ f2g = &gfx_v8_kfd2kgd;
+ break;
+#endif
+#ifdef CONFIG_DRM_AMDGPU_CIK
+ case CHIP_HAWAII:
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &hawaii_device_info;
+ f2g = &gfx_v7_kfd2kgd;
+ break;
+#endif
+ case CHIP_TONGA:
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &tonga_device_info;
+ f2g = &gfx_v8_kfd2kgd;
+ break;
+ case CHIP_FIJI:
+ if (vf)
+ device_info = &fiji_vf_device_info;
+ else
+ device_info = &fiji_device_info;
+ f2g = &gfx_v8_kfd2kgd;
+ break;
+ case CHIP_POLARIS10:
+ if (vf)
+ device_info = &polaris10_vf_device_info;
+ else
+ device_info = &polaris10_device_info;
+ f2g = &gfx_v8_kfd2kgd;
+ break;
+ case CHIP_POLARIS11:
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &polaris11_device_info;
+ f2g = &gfx_v8_kfd2kgd;
+ break;
+ case CHIP_POLARIS12:
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &polaris12_device_info;
+ f2g = &gfx_v8_kfd2kgd;
+ break;
+ case CHIP_VEGAM:
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &vegam_device_info;
+ f2g = &gfx_v8_kfd2kgd;
+ break;
+ default:
+ switch (adev->ip_versions[GC_HWIP][0]) {
+ case IP_VERSION(9, 0, 1):
+ if (vf)
+ device_info = &vega10_vf_device_info;
+ else
+ device_info = &vega10_device_info;
+ f2g = &gfx_v9_kfd2kgd;
+ break;
+#ifdef KFD_SUPPORT_IOMMU_V2
+ case IP_VERSION(9, 1, 0):
+ case IP_VERSION(9, 2, 2):
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &raven_device_info;
+ f2g = &gfx_v9_kfd2kgd;
+ break;
+#endif
+ case IP_VERSION(9, 2, 1):
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &vega12_device_info;
+ f2g = &gfx_v9_kfd2kgd;
+ break;
+ case IP_VERSION(9, 3, 0):
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &renoir_device_info;
+ f2g = &gfx_v9_kfd2kgd;
+ break;
+ case IP_VERSION(9, 4, 0):
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &vega20_device_info;
+ f2g = &gfx_v9_kfd2kgd;
+ break;
+ case IP_VERSION(9, 4, 1):
+ device_info = &arcturus_device_info;
+ f2g = &arcturus_kfd2kgd;
+ break;
+ case IP_VERSION(9, 4, 2):
+ device_info = &aldebaran_device_info;
+ f2g = &aldebaran_kfd2kgd;
+ break;
+ case IP_VERSION(10, 1, 10):
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &navi10_device_info;
+ f2g = &gfx_v10_kfd2kgd;
+ break;
+ case IP_VERSION(10, 1, 2):
+ device_info = &navi12_device_info;
+ f2g = &gfx_v10_kfd2kgd;
+ break;
+ case IP_VERSION(10, 1, 1):
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &navi14_device_info;
+ f2g = &gfx_v10_kfd2kgd;
+ break;
+ case IP_VERSION(10, 1, 3):
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &cyan_skillfish_device_info;
+ f2g = &gfx_v10_kfd2kgd;
+ break;
+ case IP_VERSION(10, 3, 0):
+ device_info = &sienna_cichlid_device_info;
+ f2g = &gfx_v10_3_kfd2kgd;
+ break;
+ case IP_VERSION(10, 3, 2):
+ device_info = &navy_flounder_device_info;
+ f2g = &gfx_v10_3_kfd2kgd;
+ break;
+ case IP_VERSION(10, 3, 1):
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &vangogh_device_info;
+ f2g = &gfx_v10_3_kfd2kgd;
+ break;
+ case IP_VERSION(10, 3, 4):
+ device_info = &dimgrey_cavefish_device_info;
+ f2g = &gfx_v10_3_kfd2kgd;
+ break;
+ case IP_VERSION(10, 3, 5):
+ device_info = &beige_goby_device_info;
+ f2g = &gfx_v10_3_kfd2kgd;
+ break;
+ case IP_VERSION(10, 3, 3):
+ if (vf)
+ device_info = NULL;
+ else
+ device_info = &yellow_carp_device_info;
+ f2g = &gfx_v10_3_kfd2kgd;
+ break;
+ default:
+ return NULL;
+ }
+ break;
}
- device_info = kfd_supported_devices[asic_type][vf];
- f2g = kfd2kgd_funcs[asic_type];
-
if (!device_info || !f2g) {
dev_err(kfd_device, "%s %s not supported in kfd\n",
- amdgpu_asic_name[asic_type], vf ? "VF" : "");
+ amdgpu_asic_name[adev->asic_type], vf ? "VF" : "");
return NULL;
}
@@ -708,20 +822,6 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
if (!kfd)
return NULL;
- /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.
- * 32 and 64-bit requests are possible and must be
- * supported.
- */
- kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd);
- if (device_info->needs_pci_atomics &&
- !kfd->pci_atomic_requested) {
- dev_info(kfd_device,
- "skipped device %x:%x, PCI rejects atomics\n",
- pdev->vendor, pdev->device);
- kfree(kfd);
- return NULL;
- }
-
kfd->kgd = kgd;
kfd->device_info = device_info;
kfd->pdev = pdev;
@@ -821,6 +921,23 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd
- kfd->vm_info.first_vmid_kfd + 1;
+ /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.
+ * 32 and 64-bit requests are possible and must be
+ * supported.
+ */
+ kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kfd->kgd);
+ if (!kfd->pci_atomic_requested &&
+ kfd->device_info->needs_pci_atomics &&
+ (!kfd->device_info->no_atomic_fw_version ||
+ kfd->mec_fw_version < kfd->device_info->no_atomic_fw_version)) {
+ dev_info(kfd_device,
+ "skipped device %x:%x, PCI rejects atomics %d<%d\n",
+ kfd->pdev->vendor, kfd->pdev->device,
+ kfd->mec_fw_version,
+ kfd->device_info->no_atomic_fw_version);
+ return false;
+ }
+
/* Verify module parameters regarding mapped process number*/
if ((hws_max_conc_proc < 0)
|| (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) {
@@ -959,7 +1076,6 @@ out:
void kgd2kfd_device_exit(struct kfd_dev *kfd)
{
if (kfd->init_complete) {
- svm_migrate_fini((struct amdgpu_device *)kfd->kgd);
device_queue_manager_uninit(kfd->dqm);
kfd_interrupt_exit(kfd);
kfd_topology_remove_device(kfd);
@@ -1057,30 +1173,28 @@ int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)
return ret;
}
-static int kfd_resume(struct kfd_dev *kfd)
+int kgd2kfd_resume_iommu(struct kfd_dev *kfd)
{
int err = 0;
err = kfd_iommu_resume(kfd);
- if (err) {
+ if (err)
dev_err(kfd_device,
"Failed to resume IOMMU for device %x:%x\n",
kfd->pdev->vendor, kfd->pdev->device);
- return err;
- }
+ return err;
+}
+
+static int kfd_resume(struct kfd_dev *kfd)
+{
+ int err = 0;
err = kfd->dqm->ops.start(kfd->dqm);
- if (err) {
+ if (err)
dev_err(kfd_device,
"Error starting queue manager for device %x:%x\n",
kfd->pdev->vendor, kfd->pdev->device);
- goto dqm_start_error;
- }
-
- return err;
-dqm_start_error:
- kfd_iommu_suspend(kfd);
return err;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
index 12d91e53556c..543e7ea75593 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
@@ -231,7 +231,7 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev,
if (sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST &&
sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) {
kfd_signal_poison_consumed_event(dev, pasid);
- amdgpu_amdkfd_gpu_reset(dev->kgd);
+ amdgpu_amdkfd_ras_poison_consumption_handler(dev->kgd);
return;
}
break;
@@ -253,7 +253,7 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev,
kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
} else if (source_id == SOC15_INTSRC_SDMA_ECC) {
kfd_signal_poison_consumed_event(dev, pasid);
- amdgpu_amdkfd_gpu_reset(dev->kgd);
+ amdgpu_amdkfd_ras_poison_consumption_handler(dev->kgd);
return;
}
} else if (client_id == SOC15_IH_CLIENTID_VMC ||
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index dab290a4d19d..f53e17a94ad8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -891,11 +891,17 @@ int svm_migrate_init(struct amdgpu_device *adev)
pgmap->ops = &svm_migrate_pgmap_ops;
pgmap->owner = SVM_ADEV_PGMAP_OWNER(adev);
pgmap->flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
+
+ /* Device manager releases device-specific resources, memory region and
+ * pgmap when driver disconnects from device.
+ */
r = devm_memremap_pages(adev->dev, pgmap);
if (IS_ERR(r)) {
pr_err("failed to register HMM device memory\n");
- devm_release_mem_region(adev->dev, res->start,
- res->end - res->start + 1);
+
+ /* Disable SVM support capability */
+ pgmap->type = 0;
+ devm_release_mem_region(adev->dev, res->start, resource_size(res));
return PTR_ERR(r);
}
@@ -908,12 +914,3 @@ int svm_migrate_init(struct amdgpu_device *adev)
return 0;
}
-
-void svm_migrate_fini(struct amdgpu_device *adev)
-{
- struct dev_pagemap *pgmap = &adev->kfd.dev->pgmap;
-
- devm_memunmap_pages(adev->dev, pgmap);
- devm_release_mem_region(adev->dev, pgmap->range.start,
- pgmap->range.end - pgmap->range.start + 1);
-}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
index 0de76b5d4973..2f5b3394c9ed 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h
@@ -47,7 +47,6 @@ unsigned long
svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr);
int svm_migrate_init(struct amdgpu_device *adev);
-void svm_migrate_fini(struct amdgpu_device *adev);
#else
@@ -55,10 +54,6 @@ static inline int svm_migrate_init(struct amdgpu_device *adev)
{
return 0;
}
-static inline void svm_migrate_fini(struct amdgpu_device *adev)
-{
- /* empty */
-}
#endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index ab83b0de6b22..6d8f9bb2d905 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -207,6 +207,7 @@ struct kfd_device_info {
bool supports_cwsr;
bool needs_iommu_device;
bool needs_pci_atomics;
+ uint32_t no_atomic_fw_version;
unsigned int num_sdma_engines;
unsigned int num_xgmi_sdma_engines;
unsigned int num_sdma_queues_per_engine;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 9fc8021bb0ab..179080329af8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -118,6 +118,13 @@ static void svm_range_remove_notifier(struct svm_range *prange)
mmu_interval_notifier_remove(&prange->notifier);
}
+static bool
+svm_is_valid_dma_mapping_addr(struct device *dev, dma_addr_t dma_addr)
+{
+ return dma_addr && !dma_mapping_error(dev, dma_addr) &&
+ !(dma_addr & SVM_RANGE_VRAM_DOMAIN);
+}
+
static int
svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,
unsigned long offset, unsigned long npages,
@@ -139,8 +146,7 @@ svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,
addr += offset;
for (i = 0; i < npages; i++) {
- if (WARN_ONCE(addr[i] && !dma_mapping_error(dev, addr[i]),
- "leaking dma mapping\n"))
+ if (svm_is_valid_dma_mapping_addr(dev, addr[i]))
dma_unmap_page(dev, addr[i], PAGE_SIZE, dir);
page = hmm_pfn_to_page(hmm_pfns[i]);
@@ -209,7 +215,7 @@ void svm_range_dma_unmap(struct device *dev, dma_addr_t *dma_addr,
return;
for (i = offset; i < offset + npages; i++) {
- if (!dma_addr[i] || dma_mapping_error(dev, dma_addr[i]))
+ if (!svm_is_valid_dma_mapping_addr(dev, dma_addr[i]))
continue;
pr_debug("dma unmapping 0x%llx\n", dma_addr[i] >> PAGE_SHIFT);
dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
@@ -1165,7 +1171,7 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
unsigned long last_start;
int last_domain;
int r = 0;
- int64_t i;
+ int64_t i, j;
last_start = prange->start + offset;
@@ -1178,7 +1184,11 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
for (i = offset; i < offset + npages; i++) {
last_domain = dma_addr[i] & SVM_RANGE_VRAM_DOMAIN;
dma_addr[i] &= ~SVM_RANGE_VRAM_DOMAIN;
- if ((prange->start + i) < prange->last &&
+
+ /* Collect all pages in the same address range and memory domain
+ * that can be mapped with a single call to update mapping.
+ */
+ if (i < offset + npages - 1 &&
last_domain == (dma_addr[i + 1] & SVM_RANGE_VRAM_DOMAIN))
continue;
@@ -1201,6 +1211,10 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
NULL, dma_addr,
&vm->last_update,
&table_freed);
+
+ for (j = last_start - prange->start; j <= i; j++)
+ dma_addr[j] |= last_domain;
+
if (r) {
pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start);
goto out;
@@ -1293,7 +1307,7 @@ struct svm_validate_context {
struct svm_range *prange;
bool intr;
unsigned long bitmap[MAX_GPU_INSTANCE];
- struct ttm_validate_buffer tv[MAX_GPU_INSTANCE+1];
+ struct ttm_validate_buffer tv[MAX_GPU_INSTANCE];
struct list_head validate_list;
struct ww_acquire_ctx ticket;
};
@@ -1320,11 +1334,6 @@ static int svm_range_reserve_bos(struct svm_validate_context *ctx)
ctx->tv[gpuidx].num_shared = 4;
list_add(&ctx->tv[gpuidx].head, &ctx->validate_list);
}
- if (ctx->prange->svm_bo && ctx->prange->ttm_res) {
- ctx->tv[MAX_GPU_INSTANCE].bo = &ctx->prange->svm_bo->bo->tbo;
- ctx->tv[MAX_GPU_INSTANCE].num_shared = 1;
- list_add(&ctx->tv[MAX_GPU_INSTANCE].head, &ctx->validate_list);
- }
r = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->validate_list,
ctx->intr, NULL);
generated by cgit v1.2.3 (git 2.25.1) at 2025-03-21 12:14:25 +0000