From 0d30e6cd3bc38073bff40edb58fa64b640bd2190 Mon Sep 17 00:00:00 2001 From: neuron-code-sharing-robot Date: Mon, 19 Oct 2026 19:14:44 +0000 Subject: [PATCH] Extracted contents of aws-neuronx-dkms-2.31.7.0.noarch.rpm --- Kbuild | 13 +- aws-neuronx-dkms-mkdeb/debian/postinst | 4 +- aws-neuronx-dkms-mkdeb/debian/prerm | 6 +- dkms.conf | 2 +- neuron_arch.c | 16 - neuron_arch.h | 28 +- neuron_cdev.c | 395 +-- neuron_cdev.h | 11 +- neuron_cinit.c | 10 +- neuron_cinit.h | 1 + neuron_core.c | 1 + neuron_core.h | 1 + neuron_crwl.c | 1 + neuron_crwl.h | 1 + neuron_device.h | 23 + neuron_dhal.c | 1 + neuron_dhal.h | 33 +- neuron_dma.c | 2595 ++++++++++++------ neuron_dma.h | 179 +- neuron_dma_diag.c | 306 +++ neuron_dma_diag.h | 71 + neuron_dmabuf.c | 1 + neuron_dmabuf.h | 1 + neuron_ds.c | 1 + neuron_ds.h | 3 +- neuron_fw_io.c | 110 +- neuron_fw_io.h | 48 +- neuron_fw_layout.c | 423 +++ neuron_fw_layout.h | 197 ++ neuron_fw_metrics_common.c | 20 + neuron_fw_metrics_common.h | 28 + neuron_ham_throttle.c | 369 +++ neuron_ham_throttle.h | 61 + neuron_ioctl.h | 50 +- neuron_log.c | 1 + neuron_log.h | 1 + neuron_mc_handle.c | 1 + neuron_mc_handle.h | 1 + neuron_mempool.c | 7 +- neuron_mempool.h | 6 +- neuron_metrics.c | 2 + neuron_metrics.h | 1 + neuron_mmap.c | 1 + neuron_mmap.h | 1 + neuron_module.c | 9 +- neuron_nq.c | 109 + neuron_nq.h | 53 + neuron_p2p.c | 1 + neuron_p2p.h | 2 +- neuron_pci.c | 60 +- neuron_pci.h | 21 +- neuron_pid.c | 1 + neuron_pid.h | 1 + neuron_platform.c | 146 + neuron_platform.h | 160 ++ neuron_power.c | 1 + neuron_power.h | 2 +- neuron_reg_access.c | 1 + neuron_reg_access.h | 1 + neuron_reset.c | 4 + neuron_reset.h | 1 + neuron_ring.c | 15 +- neuron_ring.h | 86 +- neuron_sysfs_metrics.c | 212 +- neuron_sysfs_metrics.h | 16 +- neuron_sysfs_platform.c | 308 +++ neuron_test.c | 1 + neuron_test.h | 1 + neuron_topsp.c | 1 + neuron_topsp.h | 1 + neuron_trace.h | 1 + postinstall | 2 +- postremove | 2 +- share/neuron_driver_shared.h | 15 +- share/neuron_driver_shared_tensor_batch_op.h | 1 + udma/udma.h | 28 + udma/udma_iofic.c | 25 +- udma/udma_m2m.c | 1 + udma/udma_main.c | 1 + udma/udma_regs.h | 11 +- v2/address_map.h | 1 + v2/neuron_dhal_v2.c | 330 ++- v2/notific.c | 1 + v2/notific.h | 1 + v2/sdma.h | 1 + v3/address_map.h | 1 + v3/neuron_dhal_v3.c | 493 +++- v3/neuron_fw_metrics.c | 25 + v3/neuron_fw_metrics.h | 17 + v3/neuron_pelect.c | 87 +- v3/neuron_pelect.h | 1 + v3/notific.c | 1 + v3/notific.h | 1 + v3/sdma.h | 1 + v4/address_map.h | 1 + v4/neuron_dhal_v4.c | 373 ++- v4/neuron_fw_metrics.c | 25 + v4/neuron_fw_metrics.h | 17 + vc/neuron_dhal_vc.c | 1 + 99 files changed, 6159 insertions(+), 1552 deletions(-) create mode 100644 neuron_dma_diag.c create mode 100644 neuron_dma_diag.h create mode 100644 neuron_fw_layout.c create mode 100644 neuron_fw_layout.h create mode 100644 neuron_fw_metrics_common.c create mode 100644 neuron_fw_metrics_common.h create mode 100644 neuron_ham_throttle.c create mode 100644 neuron_ham_throttle.h create mode 100644 neuron_platform.c create mode 100644 neuron_platform.h create mode 100644 neuron_sysfs_platform.c create mode 100644 v3/neuron_fw_metrics.c create mode 100644 v3/neuron_fw_metrics.h create mode 100644 v4/neuron_fw_metrics.c create mode 100644 v4/neuron_fw_metrics.h diff --git a/Kbuild b/Kbuild index 91ad5c4..36eeb3b 100644 --- a/Kbuild +++ b/Kbuild @@ -1,23 +1,26 @@ obj-m += neuron.o -neuron-objs := neuron_arch.o neuron_dhal.o +neuron-objs := neuron_arch.o neuron_platform.o neuron_dhal.o neuron-objs += neuron_reg_access.o -neuron-objs += neuron_module.o neuron_pci.o neuron_mempool.o neuron_dma.o neuron_ring.o neuron_ds.o +neuron-objs += neuron_module.o neuron_pci.o neuron_mempool.o neuron_dma.o neuron_dma_diag.o neuron_ring.o neuron_ds.o neuron-objs += neuron_core.o neuron_crwl.o neuron_cdev.o neuron_topsp.o neuron_pid.o neuron-objs += neuron_reset.o neuron_cinit.o neuron_mmap.o neuron_p2p.o neuron-objs += neuron_nq.o neuron-objs += neuron_mc_handle.o -neuron-objs += neuron_metrics.o neuron_sysfs_metrics.o +neuron-objs += neuron_metrics.o neuron_sysfs_metrics.o neuron_sysfs_platform.o neuron-objs += udma/udma_iofic.o udma/udma_m2m.o udma/udma_main.o neuron-objs += neuron_fw_io.o +neuron-objs += neuron_ham_throttle.o +neuron-objs += neuron_fw_layout.o neuron_fw_metrics_common.o neuron-objs += neuron_dmabuf.o neuron-objs += neuron_log.o neuron-objs += neuron_power.o neuron-objs += neuron_test.o neuron-objs += vc/neuron_dhal_vc.o neuron-objs += v2/notific.o v2/neuron_dhal_v2.o -neuron-objs += v3/notific.o v3/neuron_dhal_v3.o v3/neuron_pelect.o -neuron-objs += v4/neuron_dhal_v4.o +neuron-objs += v3/notific.o v3/neuron_dhal_v3.o v3/neuron_pelect.o v3/neuron_fw_metrics.o +neuron-objs += v4/neuron_dhal_v4.o v4/neuron_fw_metrics.o ccflags-y += -O3 -Wall -Werror -Wno-declaration-after-statement -Wunused-macros -Wunused-local-typedefs ccflags-y += -I$(src)/ ccflags-y += $(call cc-option,-march=armv8.2-a) + diff --git a/aws-neuronx-dkms-mkdeb/debian/postinst b/aws-neuronx-dkms-mkdeb/debian/postinst index e189062..bf41bba 100755 --- a/aws-neuronx-dkms-mkdeb/debian/postinst +++ b/aws-neuronx-dkms-mkdeb/debian/postinst @@ -1,4 +1,4 @@ -#!/usr/bin/sh +#!/bin/sh # Copyright (C) 2002-2005 Flavio Stanchina # Copyright (C) 2005-2006 Aric Cyr # Copyright (C) 2007 Mario Limonciello @@ -6,7 +6,7 @@ set -e -NAME=MODULE_NAME +NAME=aws-neuronx PACKAGE_NAME=$NAME-dkms DEB_NAME=$(echo $PACKAGE_NAME | sed 's,_,-,') CVERSION=`dpkg-query -W -f='${Version}' $DEB_NAME | awk -F "-" '{print $1}' | cut -d\: -f2` diff --git a/aws-neuronx-dkms-mkdeb/debian/prerm b/aws-neuronx-dkms-mkdeb/debian/prerm index 61b816d..e3747b1 100755 --- a/aws-neuronx-dkms-mkdeb/debian/prerm +++ b/aws-neuronx-dkms-mkdeb/debian/prerm @@ -1,7 +1,7 @@ -#!/usr/bin/sh +#!/bin/sh -NAME=MODULE_NAME -VERSION=MODULE_VERSION +NAME=aws-neuronx +VERSION=2.31.7.0 set -e diff --git a/dkms.conf b/dkms.conf index f09b0f1..36e3120 100644 --- a/dkms.conf +++ b/dkms.conf @@ -1,5 +1,5 @@ PACKAGE_NAME=aws-neuronx -PACKAGE_VERSION=2.30.2.0 +PACKAGE_VERSION=2.31.7.0 BUILT_MODULE_NAME[0]="neuron" MAKE[0]="make -C ${kernel_source_dir} M=${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/build" CLEAN="make -C ${kernel_source_dir} M=${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/build clean" diff --git a/neuron_arch.c b/neuron_arch.c index 04f77ad..5e6fce6 100644 --- a/neuron_arch.c +++ b/neuron_arch.c @@ -91,19 +91,3 @@ int narch_get_instance_type_name(char *instance_type_name, size_t instance_type_ #endif } -enum neuron_platform_type narch_detect_platform_type(const struct neuron_platform_lookup *table) { - char buf[128]; - int i; - - if (narch_get_instance_type_name(buf, sizeof(buf))) { - return NEURON_PLATFORM_TYPE_STD; - } - - for (i = 0; table[i].inst_name != NULL; i++) { - if (strncmp(buf, table[i].inst_name, strlen(table[i].inst_name)) == 0) { - return table[i].platform_type; - } - } - - return NEURON_PLATFORM_TYPE_STD; -} diff --git a/neuron_arch.h b/neuron_arch.h index 3d4b4cc..37d244d 100644 --- a/neuron_arch.h +++ b/neuron_arch.h @@ -17,24 +17,6 @@ enum neuron_arch { NEURON_ARCH_NUM }; -enum neuron_platform_type { - NEURON_PLATFORM_TYPE_STD = 0, - NEURON_PLATFORM_TYPE_ULTRASERVER = 1, - NEURON_PLATFORM_TYPE_PDS = 2, - NEURON_PLATFORM_TYPE_MAX = 3, - NEURON_PLATFORM_TYPE_INVALID, -}; - -struct neuron_platform_lookup { - const char *inst_name; - enum neuron_platform_type platform_type; -}; - -enum neuron_platform_operation_type { - NEURON_PLATFORM_OP_TYPE_DEVOPEN = 0, - NEURON_PLATFORM_OP_TYPE_EXEC = 1, -}; - /** * narch_init() - Set neuron devices architecture and revision. * @@ -89,13 +71,5 @@ bool narch_is_emu(void); */ int narch_get_instance_type_name(char *instance_type_name, size_t instance_type_name_size); -/** - * narch_detect_platform_type() - Detect platform type from instance name. - * - * @table: Array of instance name to platform type mappings. - * - * Return: Matching platform type, or NEURON_PLATFORM_TYPE_STD if no match. - */ -enum neuron_platform_type narch_detect_platform_type(const struct neuron_platform_lookup *table); - #endif + diff --git a/neuron_cdev.c b/neuron_cdev.c index dd6f776..dfc639b 100644 --- a/neuron_cdev.c +++ b/neuron_cdev.c @@ -46,7 +46,16 @@ static dev_t neuron_dev; static int major; + +static struct class neuron_dev_class_obj = { + .name = "neuron_device", + .class_groups = neuron_class_groups, +#if (!defined(RHEL_RELEASE_CODE) && (LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0))) || (defined(RHEL_RELEASE_CODE) && (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(9, 2))) + .owner = THIS_MODULE, +#endif +}; static struct class *neuron_dev_class; +static bool neuron_dev_class_registered; /* one device node per device */ #define NEURON_MAX_DEV_NODES MAX_NEURON_DEVICE_COUNT @@ -861,6 +870,8 @@ static int ncdev_memset(struct neuron_device *nd, unsigned int cmd, void *param) static int ncdev_mem_copy(struct neuron_device *nd, unsigned int cmd, void *param) { static_assert(NEURON_IOCTL_MEM_COPY != NEURON_IOCTL_MEM_COPY64); + static_assert(NEURON_IOCTL_MEM_COPY64_DEPRECATED != NEURON_IOCTL_MEM_COPY64); + static_assert(NEURON_IOCTL_MEM_COPY64 != NEURON_IOCTL_MEM_COPY64Q); struct mem_chunk *src_mc; struct mem_chunk *dst_mc; @@ -871,6 +882,8 @@ static int ncdev_mem_copy(struct neuron_device *nd, unsigned int cmd, void *para u64 size; u64 sequence_num = 0; void *context = NULL; + u32 nc_id; + int qid = NEURON_DMA_H2T_DEFAULT_QID; int ret; if (cmd == NEURON_IOCTL_MEM_COPY) { @@ -905,6 +918,19 @@ static int ncdev_mem_copy(struct neuron_device *nd, unsigned int cmd, void *para size = arg.size; sequence_num = arg.sequence_num; context = arg.context; + } else if (cmd == NEURON_IOCTL_MEM_COPY64Q) { + struct neuron_ioctl_mem_copy64q arg; + ret = neuron_copy_from_user(__func__, &arg, (struct neuron_ioctl_mem_copy64q *)param, sizeof(arg)); + if (ret) + return ret; + src_mem_handle = arg.src_mem_handle; + dst_mem_handle = arg.dst_mem_handle; + src_offset = arg.src_offset; + dst_offset = arg.dst_offset; + size = arg.size; + sequence_num = arg.sequence_num; + context = arg.context; + qid = arg.qid; } else { return -EINVAL; } @@ -924,20 +950,28 @@ static int ncdev_mem_copy(struct neuron_device *nd, unsigned int cmd, void *para pr_err("dst offset+size is too large for mem handle\n"); return -E2BIG; } - ret = ndma_memcpy_mc(nd, src_mc, dst_mc, src_offset, dst_offset, size); - if (ret) { - return ret; + // set/check qid + nc_id = ndma_mc_pair_to_nc(src_mc, dst_mc); + qid = (qid == NEURON_DMA_H2T_DEFAULT_QID) ? ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(nc_id) : qid; + if (!ndmar_qid_valid(qid)) { + pr_err("nd%02d: invalid h2t queue index %d", nd->device_index, qid); + return -ENOENT; } - if (sequence_num != 0) { - u32 nc_id = ndma_mc_pair_to_nc(src_mc, dst_mc); - int qid = ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(nc_id); - int submit_ret; - // Temporary fake async copy: the copy is already done, but report its result through CQE in queue order. - submit_ret = ndma_zerocopy_submit_completed(nd, nc_id, qid, sequence_num, 0, context); - if (submit_ret) { - return submit_ret; + if (ndma_zca_mc_copy_available(nd, src_mc, dst_mc)) { + // the copy goes through the pipelined dma path, which handles both the sync + // (sequence_num == 0) and async cases natively. + ret = ndma_zca_mc_copy_submit(nd, src_mc, dst_mc, src_offset, dst_offset, size, qid, sequence_num, context); + } else { + if (sequence_num != 0) { + pr_err("nd%02d: sequence number cannot be used on platformw w/o async zero copy", nd->device_index); + return -EOPNOTSUPP; + } + if (qid != ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(nc_id)) { + pr_err("nd%02d: invalid h2t queue index %d", nd->device_index, qid); + return -ENOENT; } + ret = ndma_memcpy_mc(nd, src_mc, dst_mc, src_offset, dst_offset, size); } trace_ioctl_mem_copy(nd, src_mc, dst_mc); return ret; @@ -1332,6 +1366,7 @@ static int ncdev_mem_buf_zerocopy64(struct neuron_device *nd, unsigned int cmd, // limit to internal threshold to prevent DoS attack bar4_wr_threshold = (arg.bar4_wr_threshold < BAR4_WR_THRESHOLD_MAX) ? arg.bar4_wr_threshold : BAR4_WR_THRESHOLD_MAX; use_bar4_wr = !narch_is_qemu() && + !narch_is_emu() && (size <= bar4_wr_threshold) && copy_to_mem_handle && nd->npdev.bar4_pa && @@ -1359,9 +1394,11 @@ static int ncdev_mem_buf_zerocopy64(struct neuron_device *nd, unsigned int cmd, // for async mode, BAR4 write is already done; enqueue a completed ctx to preserve CQE order. // for sync mode, return the BAR4 write result directly - if (sequence_num != 0) { - ret = ndma_zerocopy_submit_completed(nd, nc_id, qid, sequence_num, ret, context); - } + if ((sequence_num != 0) && !ret) { + bool bar4_wr = true; + ret = ndma_zca_zerocopy_submit(nd, nc_id, NULL, 1, 0ull, qid, true, + bar4_wr, sequence_num, context); + } // TODO we could create a failed request if sequence_num is non-zero return ret; } else { nrt_tensor_batch_op_t op; @@ -1375,10 +1412,10 @@ static int ncdev_mem_buf_zerocopy64(struct neuron_device *nd, unsigned int cmd, op.offset = offset; op.buffer = buffer; op.size = size; - - ret = ndma_zerocopy_submit(nd, nc_id, &op, 1, dev_base, qid, - copy_to_mem_handle ? true : false, - sequence_num, context); + + bool bar4_wr = false; + ret = ndma_zca_zerocopy_submit(nd, nc_id, &op, 1, dev_base, qid, copy_to_mem_handle ? true : false, + bar4_wr, sequence_num, context); return ret; } } @@ -1425,6 +1462,7 @@ static int ncdev_mem_buf_zerocopy64_batch(struct neuron_device *nd, void *param) goto cleanup; } + // sanity check each batch entry and count up total ops for (i = 0; i < arg.num_batches; i++) { neuron_memcpy_batch_t batch = batches[i]; size_t num_ops = batch.num_ops; @@ -1470,7 +1508,7 @@ static int ncdev_mem_buf_zerocopy64_batch(struct neuron_device *nd, void *param) pr_err("failed to copy ops from user space\n"); goto cleanup; } - batch->ops_ptr = ops; + batch->ops_ptr = ops; // TODO note this is transmuting the batch ops pointer in the copy to change from pointing to user ops to driver ops ops_buffer_offset += num_ops; mc = ncdev_mem_handle_to_mem_chunk(nd, batch->mem_handle); @@ -1481,7 +1519,7 @@ static int ncdev_mem_buf_zerocopy64_batch(struct neuron_device *nd, void *param) } bar4_wr_threshold = (batch->bar4_wr_threshold < BAR4_WR_THRESHOLD_MAX) ? batch->bar4_wr_threshold : BAR4_WR_THRESHOLD_MAX; - use_bar4_wr = !narch_is_qemu() && arg.is_copy_to_device && nd->npdev.bar4_pa && mc->mem_location == MEM_LOC_DEVICE; + use_bar4_wr = !narch_is_qemu() && !narch_is_emu() && arg.is_copy_to_device && nd->npdev.bar4_pa && mc->mem_location == MEM_LOC_DEVICE; for (j = 0; j < batch->num_ops; j++) { nrt_tensor_batch_op_t *op = &ops[j]; @@ -1539,6 +1577,8 @@ static int ncdev_mem_buf_zerocopy64_batch(struct neuron_device *nd, void *param) } } } + + //FIXME - there is a disconnect each batch has it's own context but we only have one sequence number. } else { u32 nc_id = ndma_mc_pair_to_nc(mc, mc); int qid = (arg.h2t_qid == NEURON_DMA_H2T_DEFAULT_QID) ? ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(nc_id) : arg.h2t_qid; @@ -1557,7 +1597,9 @@ static int ncdev_mem_buf_zerocopy64_batch(struct neuron_device *nd, void *param) } // use the zero-copy batch function for ops within a single batch - ret = ndma_zerocopy_submit(nd, nc_id, batch->ops_ptr, batch->num_ops, dev_base, qid, arg.is_copy_to_device, arg.sequence_num, NULL); + bool bar4_wr = false; // + ret = ndma_zca_zerocopy_submit(nd, nc_id, batch->ops_ptr, batch->num_ops, dev_base, qid, arg.is_copy_to_device, + bar4_wr, arg.sequence_num, batch->context); if (ret) { pr_err("batch zero-copy DMA failed on batch %d on nd%02d: %d\n", i, nd->device_index, ret); goto cleanup; @@ -1573,6 +1615,41 @@ static int ncdev_mem_buf_zerocopy64_batch(struct neuron_device *nd, void *param) return ret; } +static int ncdev_h2d_desc_execute(struct neuron_device *nd, void *param) +{ + struct neuron_ioctl_h2d_desc_execute arg; + int qid; + int ret; + + ret = neuron_copy_from_user(__func__, &arg, (struct neuron_ioctl_h2d_desc_execute *)param, sizeof(arg)); + if (ret) { + return ret; + } + + ret = ncdev_ncid_valid(arg.nc_id); + if (ret) { + return ret; + } + + qid = (arg.h2t_qid == NEURON_DMA_H2T_DEFAULT_QID) + ? ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(arg.nc_id) + : arg.h2t_qid; + if (!ndmar_qid_valid(qid)) { + pr_err("nd%02d: invalid h2t queue index %d", nd->device_index, qid); + return -ENOENT; + } + + if (!ndma_zerocopy_supported()) { + pr_err_once("nd%02d: zero copy is not supported for architectures requiring DMA retry", nd->device_index); + return -EINVAL; + } + + return ndma_zca_desc_execute_submit(nd, arg.nc_id, qid, + (void *)arg.tx_descs, arg.tx_descs_bytes, + (void *)arg.rx_descs, arg.rx_descs_bytes, + arg.sequence_num, arg.context); +} + static long ncdev_semaphore_ioctl(struct neuron_device *nd, unsigned int cmd, void *param) { int ret; @@ -1631,20 +1708,24 @@ static long ncdev_bar_read(struct neuron_device *nd, u8 bar, u64 *reg_addresses, int i; if (bar == 0 || bar == 2) { u32 *data = NULL; + u64 bar_pa = (u64)nd->npdev.bar0_pa; + u64 bar_size = nd->npdev.bar0_size; + // Userspace addresses are BAR0-physical-relative (bar0_pa + offset). + // Validate against the physical BAR window, then translate each to the + // kernel VA (bar0 + offset) that the FWIO CSR reader actually dereferences. + // + // nd->npdev.bar0 is initialized to APB bar + // On V2 arch, APB bar is 0 usually, but 2 in case of QEMU + // From V2 arch onwards, nd->npdev.bar2 (axi_bar) is no longer initialized + // TODO: Remove bar2 fields for (i = 0; i < data_count; i++) { - u64 bar_addr = (u64)nd->npdev.bar0; - u64 bar_size = nd->npdev.bar0_size; - - // From V2 arch onwards, nd->npdev.bar2 (axi_bar) is no longer initialized - // TODO: Remove bar2 fields - - // nd->npdev.bar0 is initialized to APB bar - // On V2 arch, APB bar is 0 usually, but 2 in case of QEMU - - if ((reg_addresses[i] < bar_addr) || (reg_addresses[i] >= bar_addr + bar_size)) { + u64 off; + if ((reg_addresses[i] < bar_pa) || (reg_addresses[i] >= bar_pa + bar_size)) { return -EINVAL; } + off = reg_addresses[i] - bar_pa; + reg_addresses[i] = (u64)nd->npdev.bar0 + off; } data = kmalloc(data_size, GFP_KERNEL); if (data == NULL) @@ -1708,10 +1789,13 @@ static int ncdev_bar_write_data(struct neuron_device *nd, u8 bar, u64 *reg_addre if (bar == 0) { int i; for (i = 0; i < data_count; i++) { - u64 bar_addr = (u64)nd->npdev.bar0; + // Userspace addresses are BAR0-physical-relative (bar0_pa + offset). + // Validate against the physical BAR window, recover the offset, then + // write to the kernel VA (bar0 + offset). + u64 bar_pa = (u64)nd->npdev.bar0_pa; u64 bar_size = nd->npdev.bar0_size; - u64 off = reg_addresses[i] - bar_addr; - if ((reg_addresses[i] < bar_addr) || (reg_addresses[i] >= bar_addr + bar_size)) { + u64 off = reg_addresses[i] - bar_pa; + if ((reg_addresses[i] < bar_pa) || (reg_addresses[i] >= bar_pa + bar_size)) { return -EINVAL; } if (ndhal->ndhal_ndma.ndma_is_bar0_write_blocked(off)) { @@ -2019,9 +2103,9 @@ static long ncdev_device_info(struct neuron_device *nd, void *param) } result.connected_device_count = connected_device_count; - result.bar_address[0] = (u64)nd->npdev.bar0; + result.bar_address[0] = (u64)nd->npdev.bar0_pa; result.bar_size[0] = nd->npdev.bar0_size; - result.bar_address[1] = (u64)nd->npdev.bar2; + result.bar_address[1] = (u64)nd->npdev.bar2_pa; result.bar_size[1] = nd->npdev.bar2_size; ret = copy_to_user(param, &result, sizeof(result)); @@ -2060,10 +2144,14 @@ static long ncdev_driver_info(unsigned int cmd, void *param) NEURON_DRIVER_FEATURE_MEM_ALLOC_TYPE | NEURON_DRIVER_FEATURE_HBM_SCRUB | NEURON_DRIVER_FEATURE_MEM_ALLOC64 | NEURON_DRIVER_FEATURE_CONTIGUOUS_SCRATCHPAD | NEURON_DRIVER_FEATURE_ZEROCOPY | NEURON_DRIVER_FEATURE_PINNED_HOST_MEM | - NEURON_DRIVER_FEATURE_ALLOC_WITH_PA; + NEURON_DRIVER_FEATURE_ALLOC_WITH_PA | NEURON_DRIVER_FEATURE_DMABUF_V2; if (ndma_zerocopy_supported()) - driver_info.feature_flags1 |= NEURON_DRIVER_FEATURE_ASYNC_RW | - NEURON_DRIVER_FEATURE_ASYNC_COPY; + driver_info.feature_flags1 |= NEURON_DRIVER_FEATURE_ASYNC_RW | NEURON_DRIVER_FEATURE_ASYNC_COPY | + NEURON_DRIVER_FEATURE_ASYNC_COMPL | NEURON_DRIVER_FEATURE_ASYNC_MC2MC | + NEURON_DRIVER_FEATURE_H2D_DESC_EXECUTE | + NEURON_DRIVER_FEATURE_ASYNC_IO_SUPPORTED; + if (ndhal->ndhal_nq.h2d_dma_nq_nc_id_supported) + driver_info.feature_flags1 |= NEURON_DRIVER_FEATURE_H2D_DMA_NQ_NC_ID; return copy_to_user(param, &driver_info, sizeof(driver_info)); } @@ -2161,7 +2249,33 @@ static long ncdev_nc_nq_init_deprecated(struct neuron_device *nd, void *param) return copy_to_user(param, &arg, sizeof(arg)); } -static long ncdev_nc_nq_init_libnrt(struct neuron_device *nd, void *param) +/* + * For NQ_DEVICE_TYPE_H2D_DMA the ioctl's engine_index carries the caller's owned physical + * NC and the driver selects that SEngine's top-level engine. Engine ids (32+, 128+) never + * pass the NC range check, so an old runtime that still sends an engine id is rejected. + * Either core sharing the engine may allocate its queue; nnq_h2d_init grants it to the + * first process and answers others with -EBUSY. + */ +static int ncdev_h2d_nq_init(struct neuron_device *nd, u32 nc_id, u32 size, u32 on_host_memory, + u32 hbm_index, bool force_alloc_mem, struct mem_chunk **mc, u64 *mmap_offset) +{ + uint32_t h2d_eng_id; + int ret; + + ret = ncdev_ncid_valid(nc_id); + if (ret) + return ret; + if (!(ndhal->ndhal_address_map.dev_nc_map & BIT(nc_id))) + return -EINVAL; + if (!ndhal->ndhal_nq.h2d_dma_nq_nc_id_supported) + return -EOPNOTSUPP; + h2d_eng_id = ndhal->ndhal_ndmar.ndmar_get_h2t_eng_id(nd, nc_id); + + return nnq_h2d_init(nd, h2d_eng_id, nc_id, size, on_host_memory, hbm_index, + force_alloc_mem, mc, mmap_offset); +} + +static long ncdev_nc_nq_init_libnrt(struct neuron_device *nd, struct file *filep, void *param) { struct neuron_ioctl_notifications_init_v2 arg; int ret; @@ -2179,6 +2293,10 @@ static long ncdev_nc_nq_init_libnrt(struct neuron_device *nd, void *param) ret = ndhal->ndhal_topsp.ts_nq_init(nd, arg.nq_dev_id, arg.engine_index, arg.nq_type, arg.size, arg.on_host_memory, arg.hbm_index, false, &mc, &arg.mmap_offset); + } else if (arg.nq_dev_type == NQ_DEVICE_TYPE_H2D_DMA) { + ret = ncdev_h2d_nq_init(nd, arg.engine_index, arg.size, + arg.on_host_memory, arg.hbm_index, + false, &mc, &arg.mmap_offset); } else { return -ENOSYS; } @@ -2192,7 +2310,7 @@ static long ncdev_nc_nq_init_libnrt(struct neuron_device *nd, void *param) return copy_to_user(param, &arg, sizeof(arg)); } -static long ncdev_nc_nq_init_with_realloc_libnrt(struct neuron_device *nd, void *param) +static long ncdev_nc_nq_init_with_realloc_libnrt(struct neuron_device *nd, struct file *filep, void *param) { struct neuron_ioctl_notifications_init_with_realloc_v2 arg; int ret; @@ -2210,6 +2328,10 @@ static long ncdev_nc_nq_init_with_realloc_libnrt(struct neuron_device *nd, void ret = ndhal->ndhal_topsp.ts_nq_init(nd, arg.nq_dev_id, arg.engine_index, arg.nq_type, arg.size, arg.on_host_memory, arg.hbm_index, arg.force_alloc_mem, &mc, &arg.mmap_offset); + } else if (arg.nq_dev_type == NQ_DEVICE_TYPE_H2D_DMA) { + ret = ncdev_h2d_nq_init(nd, arg.engine_index, arg.size, + arg.on_host_memory, arg.hbm_index, + arg.force_alloc_mem, &mc, &arg.mmap_offset); } else { return -ENOSYS; } @@ -2645,6 +2767,10 @@ static int ncdev_nc_pid_state_dump(unsigned int cmd, void *param) return ret; } +// Max DMA engines used per HBM scrub. All ndma_get_engines_with_host_connectivity +// implementations report either 16 or 32 engines, so 32 bounds every case. +#define MAX_SCRUB_DMA_ENGINES 32 + static long ncdev_hbm_scrub_start(struct neuron_device *nd, void *param) { struct neuron_ioctl_hbm_scrub_start arg; int ret = neuron_copy_from_user(__func__, &arg, (struct neuron_ioctl_hbm_scrub_start *)param, @@ -2662,9 +2788,9 @@ static long ncdev_hbm_scrub_start(struct neuron_device *nd, void *param) { // crwl to allow for a bitmap of devices uint32_t num_dma_engines = 0; - uint32_t dma_engines[NUM_DMA_ENG_PER_DEVICE]; + uint32_t dma_engines[MAX_SCRUB_DMA_ENGINES]; ndhal->ndhal_ndma.ndma_get_engines_with_host_connectivity(arg.hbm_index, dma_engines, &num_dma_engines); - if (num_dma_engines != 16) { + if (num_dma_engines != 16 && num_dma_engines != 32) { // Any power of 2 should be fine if we update array sizes, but we should test before enabling pr_err("HBM scrub: Unsupported number (%d) of DMA engines\n", num_dma_engines); return -1; @@ -2675,7 +2801,7 @@ static long ncdev_hbm_scrub_start(struct neuron_device *nd, void *param) { struct mem_chunk **completion_mc_ptr = &(nd->hbm_scrub_ctx.completion_marker_buf[arg.hbm_index]); struct mem_chunk **hostbuf_mc = nd->hbm_scrub_ctx.hostbuf_mc; *completion_mc_ptr = NULL; - void **completion_bufs[16]; + void **completion_bufs[MAX_SCRUB_DMA_ENGINES]; uint32_t *buf = NULL; int i = 0; for (i = 0; i < num_dma_engines; i++) { @@ -2704,8 +2830,8 @@ static long ncdev_hbm_scrub_start(struct neuron_device *nd, void *param) { uint64_t curaddr = startaddr; const uint64_t endaddr = ndhal->ndhal_mpset.device_dram_end_addr[arg.hbm_index]; - uint32_t num_descs[16] = {0}; - uint32_t estimated_descs = (endaddr - startaddr) / (transfer_size * 16) + 3; // some extra for host transfers and completion marker + uint32_t num_descs[MAX_SCRUB_DMA_ENGINES] = {0}; + uint32_t estimated_descs = (endaddr - startaddr) / (transfer_size * num_dma_engines) + 3; // some extra for host transfers and completion marker uint32_t allocated_descs = ndmar_ring_get_desc_count(estimated_descs); uint64_t ring_size = allocated_descs * sizeof(union udma_desc); @@ -2842,9 +2968,9 @@ static long ncdev_hbm_scrub_wait_for_cmpl(struct neuron_device *nd, void *param) const uint64_t transfer_size = MAX_DMA_DESC_SIZE; int num_dma_engines = 0; - int dma_engines[NUM_DMA_ENG_PER_DEVICE]; + int dma_engines[MAX_SCRUB_DMA_ENGINES]; ndhal->ndhal_ndma.ndma_get_engines_with_host_connectivity(arg.hbm_index, dma_engines, &num_dma_engines); - if (num_dma_engines != 16) { + if (num_dma_engines != 16 && num_dma_engines != 32) { // Any power of 2 should be fine if we update array sizes, but we should test before enabling pr_err("HBM scrub: Unsupported number (%d) of DMA engines\n", num_dma_engines); return -1; @@ -2859,7 +2985,7 @@ static long ncdev_hbm_scrub_wait_for_cmpl(struct neuron_device *nd, void *param) pr_err("Scrub failed for nd%d HBM %d: scrub context corrupted\n", nd->device_index, arg.hbm_index); goto scrub_cleanup; } - void **completion_bufs[16]; + void **completion_bufs[MAX_SCRUB_DMA_ENGINES]; const uint64_t endaddr = ndhal->ndhal_mpset.device_dram_end_addr[arg.hbm_index]; const uint64_t hbm_size = endaddr - ndhal->ndhal_mpset.device_dram_effective_base_addr[arg.hbm_index]; uint64_t approx_num_desc = hbm_size / (num_dma_engines * transfer_size); @@ -2872,7 +2998,7 @@ static long ncdev_hbm_scrub_wait_for_cmpl(struct neuron_device *nd, void *param) struct ndma_eng *eng = &nd->ndma_engine[eng_id]; struct ndma_queue *queue = &eng->queues[qid]; struct ndma_ring *ring = &queue->ring_info; - ret = ndma_memcpy_wait_for_completion(eng, ring, approx_num_desc, completion_bufs[i], false, true); + ret = ndma_memcpy_wait_for_completion(eng, ring, approx_num_desc, approx_num_desc, completion_bufs[i], false, true); if (ret) { pr_err("Scrub failed for nd%d HBM %d\n", nd->device_index, arg.hbm_index); goto scrub_cleanup; @@ -3331,6 +3457,30 @@ static int ncdev_host_mem_unpin(void *param) return ndma_unpin_host_memory(arg.va); } +static int ncdev_pcie_profile(struct neuron_device *nd, void *param) +{ + struct neuron_ioctl_pcie_profile arg; + int ret; + + ret = neuron_copy_from_user(__func__, &arg, (struct neuron_ioctl_pcie_profile *)param, sizeof(arg)); + if (ret) { + return ret; + } + + if (arg.ctrl == 0) { + ret = fw_io_set_pcie_profile(nd->fw_io_ctx, arg.profile); + } else if (arg.ctrl == 1) { + ret = fw_io_get_pcie_profile(nd->fw_io_ctx, &arg.profile); + if (ret == 0) { + ret = neuron_copy_to_user(__func__, (void*)param, &arg, sizeof(arg)); + } + } else { + return -ENOTSUPP; + } + + return ret; +} + inline static long ncdev_misc_ioctl(struct file *filep, unsigned int cmd, unsigned long param) { if ((cmd == NEURON_IOCTL_CRWL_NC_RANGE_MARK) || (cmd == NEURON_IOCTL_CRWL_NC_RANGE_MARK_EXT0)) { return ncdev_crwl_nc_range_mark(filep, cmd, (void *)param); @@ -3524,6 +3674,8 @@ static long ncdev_ioctl(struct file *filep, unsigned int cmd, unsigned long para return ncdev_mem_buf_zerocopy64(nd, cmd, (void *)param); } else if (cmd == NEURON_IOCTL_MEM_BUF_ZEROCOPY64_BATCHES) { return ncdev_mem_buf_zerocopy64_batch(nd, (void *)param); + } else if (cmd == NEURON_IOCTL_H2D_DESC_EXECUTE) { + return ncdev_h2d_desc_execute(nd, (void *)param); } else if (cmd == NEURON_IOCTL_PROGRAM_ENGINE) { return ncdev_program_engine(nd, (void *)param); } else if (_IOC_NR(cmd) == _IOC_NR(NEURON_IOCTL_PROGRAM_ENGINE_NC)) { @@ -3553,9 +3705,9 @@ static long ncdev_ioctl(struct file *filep, unsigned int cmd, unsigned long para } else if (cmd == NEURON_IOCTL_NOTIFICATIONS_INIT_V1) { return ncdev_nc_nq_init_deprecated(nd, (void *)param); } else if (cmd == NEURON_IOCTL_NOTIFICATIONS_INIT_V2) { - return ncdev_nc_nq_init_libnrt(nd, (void *)param); + return ncdev_nc_nq_init_libnrt(nd, filep, (void *)param); } else if (cmd == NEURON_IOCTL_NOTIFICATIONS_INIT_WITH_REALLOC_V2) { - return ncdev_nc_nq_init_with_realloc_libnrt(nd, (void *)param); + return ncdev_nc_nq_init_with_realloc_libnrt(nd, filep, (void *)param); } else if (cmd == NEURON_IOCTL_NOTIFICATIONS_DESTROY_V1) { return 0; } else if (cmd == NEURON_IOCTL_NOTIFICATIONS_QUEUE_INFO) { @@ -3604,6 +3756,8 @@ static long ncdev_ioctl(struct file *filep, unsigned int cmd, unsigned long para return ncdev_get_async_h2d_dma_compl_queues(nd, (void*)param); } else if (cmd == NEURON_IOCTL_GET_BAR_INFO) { return ncdev_get_fw_bars(nd, (void*)param); + } else if (cmd == NEURON_IOCTL_PCIE_PROFILE) { + return ncdev_pcie_profile(nd, (void*)param); } // B/W compatibility @@ -3712,6 +3866,7 @@ static int ncdev_flush(struct file *filep, fl_owner_t id) // If this proc exited in the middle of a reset, wait for the reset to be processed. nr_wait(nd, task_tgid_nr(current), true); ndmar_handle_process_exit(nd, task_tgid_nr(current)); + nnq_h2d_handle_process_exit(nd); msleep(10); // TODO - confirm with HW dev, whether any delay needed after q reset. ncrwl_release_current_process(nd); neuron_ds_release_pid(&nd->datastore, task_tgid_nr(current)); @@ -3797,39 +3952,24 @@ static struct file_operations ncdev_fops = { .get_unmapped_area = ncdev_get_unmapped_area, }; -static ssize_t device_reset_show(struct device *dev, struct device_attribute *attr, char *buf) +void ncdev_trigger_sysfs_reset(struct neuron_device *nd) { - int minor = MINOR(dev->devt); - if (minor >= NEURON_MAX_DEV_NODES) { - return -ENODEV; - } - return scnprintf(buf, PAGE_SIZE, "%d\n", devnodes[minor].ndev->device_state); -} + struct ncdev *devnode = (struct ncdev *)nd->ncdev; -static ssize_t driver_reset_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) -{ - struct ncdev *devnode; - int minor = MINOR(dev->devt); - if (minor >= NEURON_MAX_DEV_NODES) { - return -ENODEV; - } - devnode = &devnodes[minor]; + if (!devnode) + return; mutex_lock(&devnode->ncdev_lock); if (devnode->open_count == 0) { // only trigger sysfs reset if the device is not opened by app - nr_start_ncs(devnode->ndev, NEURON_NC_MAP_DEVICE, NEURON_RESET_REQUEST_ALL); + nr_start_ncs(nd, NEURON_NC_MAP_DEVICE, NEURON_RESET_REQUEST_ALL); } mutex_unlock(&devnode->ncdev_lock); - - return count; } -static DEVICE_ATTR(reset, S_IWUSR | S_IRUSR, device_reset_show, driver_reset_store); - static ssize_t neuron_core_count_show(struct device *dev, struct device_attribute *attr, char *buf) { // We would prefer to have a newline here for consistency with other output but - // that breaks backward compatibility with the device plugin. They are fixing the + // that breaks backward compatibility with the device plugin. They are fixing the // plugin to handle if a newline is present, so in the future we can add it when // older plugins are not a concern - https://tiny.amazon.com/k1ezuoub return scnprintf(buf, PAGE_SIZE, "%d", ndhal->ndhal_address_map.nc_per_device); @@ -3837,93 +3977,9 @@ static ssize_t neuron_core_count_show(struct device *dev, struct device_attribut static DEVICE_ATTR(core_count, S_IRUGO, neuron_core_count_show, NULL); -#define CONNECTED_DEVICES_MAX_LEN 20 -static ssize_t neuron_connected_devices_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - int ret = 0; - int i = 0; - int offset = 0; - struct ncdev *devnode; - struct neuron_device *nd; - u32 connected_devices[MAX_NEURON_DEVICE_COUNT]; - int connected_device_count = 0; - - int minor = MINOR(dev->devt); // neuron device id - if (minor >= NEURON_MAX_DEV_NODES) { - return -ENODEV; - } - devnode = &devnodes[minor]; - nd = devnode->ndev; - ret = ndhal->ndhal_fw_io.fw_io_topology(nd->fw_io_ctx, nd->pdev->device, minor, connected_devices, &connected_device_count); - if (ret) - return ret; - - for (i = 0; i < connected_device_count - 1; i++) { - offset += snprintf(buf + offset, CONNECTED_DEVICES_MAX_LEN - offset, "%d, ", connected_devices[i]); - if (offset >= CONNECTED_DEVICES_MAX_LEN) { - pr_err("snprintf failed when showing connected devices\n"); - return 0; - } - } - if (connected_device_count == 0) - offset += snprintf(buf, CONNECTED_DEVICES_MAX_LEN, "\n"); - else - offset += snprintf(buf + offset, CONNECTED_DEVICES_MAX_LEN - offset, "%d\n", connected_devices[connected_device_count - 1]); - if (offset >= CONNECTED_DEVICES_MAX_LEN) { - pr_err("snprintf failed when showing connected devices\n"); - return 0; - } - - return offset; -} - -static DEVICE_ATTR(connected_devices, S_IRUGO, neuron_connected_devices_show, NULL); - -static ssize_t fw_api_version_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - struct neuron_device *nd; - int fw_api_version; - int minor = MINOR(dev->devt); - if (minor >= NEURON_MAX_DEV_NODES) { - return -ENODEV; - } - nd = devnodes[minor].ndev; - - fw_io_api_version_read(nd->npdev.bar0, &fw_api_version); - if (fw_api_version == 0xdeadbeef) { // the value is not readable during reset, try later - return scnprintf(buf, PAGE_SIZE, "busy\n"); - } - return scnprintf(buf, PAGE_SIZE, "%u\n", fw_api_version); -} - -static DEVICE_ATTR(fw_api_version, S_IRUGO, fw_api_version_show, NULL); - -static ssize_t fw_build_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - int ret; - struct neuron_device *nd; - u32 fw_build; - int minor = MINOR(dev->devt); - if (minor >= NEURON_MAX_DEV_NODES) { - return -ENODEV; - } - nd = devnodes[minor].ndev; - - ret = fw_io_fw_build_read(nd->npdev.bar0, &fw_build); - if (ret || fw_build == 0xdeadbeef) { // the value is not readable during reset, try later - return scnprintf(buf, PAGE_SIZE, "busy\n"); - } - return scnprintf(buf, PAGE_SIZE, "%u\n", fw_build); -} - -static DEVICE_ATTR(fw_build, S_IRUGO, fw_build_show, NULL); - +// Retained for backwards compatibility; all other neuron{N} nodes are managed by nsysfsmetric. static struct attribute *attrs[] = { - &dev_attr_reset.attr, &dev_attr_core_count.attr, - &dev_attr_connected_devices.attr, - &dev_attr_fw_api_version.attr, - &dev_attr_fw_build.attr, NULL, }; @@ -4267,7 +4323,7 @@ int ncdev_class_attr_init(void) int class_attrs_cnt; for (i = 0; i < sizeof(ncdev_platform_class_attrs) / sizeof(*ncdev_platform_class_attrs); i++) { - if (ncdev_platform_class_attrs[i].platform_type == ndhal->ndhal_arch.platform_type) { + if (ncdev_platform_class_attrs[i].platform_type == ndhal->ndhal_arch.platform->type) { class_attrs = ncdev_platform_class_attrs[i].class_attrs; class_attrs_cnt = ncdev_platform_class_attrs[i].class_attrs_cnt; } @@ -4301,7 +4357,7 @@ void ncdev_class_attr_cleanup(void) int class_attrs_cnt; for (i = 0; i < sizeof(ncdev_platform_class_attrs) / sizeof(*ncdev_platform_class_attrs); i++) { - if (ncdev_platform_class_attrs[i].platform_type == ndhal->ndhal_arch.platform_type) { + if (ncdev_platform_class_attrs[i].platform_type == ndhal->ndhal_arch.platform->type) { class_attrs = ncdev_platform_class_attrs[i].class_attrs; class_attrs_cnt = ncdev_platform_class_attrs[i].class_attrs_cnt; } @@ -4326,8 +4382,10 @@ static void ncdev_cleanup(void) pr_err("Error! ncdev is not NULL"); } - if (neuron_dev_class) { - class_destroy(neuron_dev_class); + if (neuron_dev_class_registered) { + class_unregister(neuron_dev_class); + neuron_dev_class_registered = false; + neuron_dev_class = NULL; } unregister_chrdev_region(MKDEV(major, 0), NEURON_MAX_DEV_NODES); @@ -4349,15 +4407,13 @@ int ncdev_module_init(void) major = MAJOR(neuron_dev); -#if (!defined(RHEL_RELEASE_CODE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0))) || (defined(RHEL_RELEASE_CODE) && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(9, 2))) - neuron_dev_class = class_create("neuron_device"); -#else - neuron_dev_class = class_create(THIS_MODULE, "neuron_device"); -#endif - if (IS_ERR(neuron_dev_class)) { - ret = PTR_ERR(neuron_dev_class); + ret = class_register(&neuron_dev_class_obj); + if (ret) { + pr_err("failed to register neuron_device class: %d\n", ret); goto fail; } + neuron_dev_class = &neuron_dev_class_obj; + neuron_dev_class_registered = true; return ret; @@ -4370,3 +4426,4 @@ void ncdev_module_exit(void) { ncdev_cleanup(); } + diff --git a/neuron_cdev.h b/neuron_cdev.h index 0e24758..da59839 100644 --- a/neuron_cdev.h +++ b/neuron_cdev.h @@ -62,8 +62,17 @@ int ncdev_module_init(void); /** * ncdev_module_exit() - Clean up and release resources associated with the character devices - * + * */ void ncdev_module_exit(void); +/** + * ncdev_trigger_sysfs_reset() - Trigger a device reset in response to a write to the + * neuron{N}/ctrl/reset sysfs node. Only resets when the device is not opened by an application. + * + * @param nd: the neuron device to reset + */ +void ncdev_trigger_sysfs_reset(struct neuron_device *nd); + #endif + diff --git a/neuron_cinit.c b/neuron_cinit.c index 6dc510f..a06b38a 100644 --- a/neuron_cinit.c +++ b/neuron_cinit.c @@ -12,6 +12,7 @@ #include "neuron_ioctl.h" #include "neuron_device.h" +#include "neuron_ham_throttle.h" #define NCI_RETRY_COUNT 10 @@ -49,13 +50,17 @@ void nci_set_state(struct neuron_device *nd, u32 nc_id, u32 state, u32 *new_stat nci_wait(nd, nc_id); } } else if (state == NEURON_CINIT_STATE_COMPLETED) { + bool complete = false; mutex_lock(&nci->nci_lock); current_state = nci->state; - if (nci->state == NEURON_CINIT_STATE_STARTED) + if (nci->state == NEURON_CINIT_STATE_STARTED) { nci->state = NEURON_CINIT_STATE_COMPLETED; - else + complete = true; + } else pr_err("nd%d nc:%d can't set init state to complete without starting\n", nd->device_index, nc_id); mutex_unlock(&nci->nci_lock); + if (complete) + neuron_ham_throttle_reprogram_nc(nd, nc_id); } else { pr_err("nd%d nc:%d invalid set init state\n", nd->device_index, nc_id); } @@ -82,3 +87,4 @@ void nci_reset_state(struct neuron_device *nd) nci_reset_state_nc(nd, i); } } + diff --git a/neuron_cinit.h b/neuron_cinit.h index 62b65ad..68b0aee 100644 --- a/neuron_cinit.h +++ b/neuron_cinit.h @@ -43,3 +43,4 @@ void nci_reset_state(struct neuron_device *nd); void nci_reset_state_nc(struct neuron_device *nd, u32 nc_id); #endif + diff --git a/neuron_core.c b/neuron_core.c index a15718e..3be2bf6 100644 --- a/neuron_core.c +++ b/neuron_core.c @@ -156,3 +156,4 @@ int nc_event_set(struct neuron_device *nd, u8 nc_id, u16 event_index, u32 value) writel(value, addr); return 0; } + diff --git a/neuron_core.h b/neuron_core.h index a2909c3..379ef02 100644 --- a/neuron_core.h +++ b/neuron_core.h @@ -122,3 +122,4 @@ void nc_nq_device_init(struct neuron_device *nd); #endif + diff --git a/neuron_crwl.c b/neuron_crwl.c index b89f302..36e641d 100644 --- a/neuron_crwl.c +++ b/neuron_crwl.c @@ -289,3 +289,4 @@ void ncrwl_release_current_process(struct neuron_device *nd) mutex_unlock(&crwl->lock); } } + diff --git a/neuron_crwl.h b/neuron_crwl.h index 720ce39..4b6a495 100644 --- a/neuron_crwl.h +++ b/neuron_crwl.h @@ -110,3 +110,4 @@ int ncrwl_writer_downgrade(struct neuron_device *nd, u32 nc_index, struct neuron void ncrwl_release_current_process(struct neuron_device *nd); #endif + diff --git a/neuron_device.h b/neuron_device.h index 8a89bf3..d0f888a 100644 --- a/neuron_device.h +++ b/neuron_device.h @@ -26,7 +26,9 @@ #include "neuron_sysfs_metrics.h" #include "neuron_log.h" #include "neuron_power.h" +#include "neuron_ham_throttle.h" #include "neuron_fw_io.h" +#include "neuron_fw_layout.h" #ifndef static_assert #define static_assert(expr, ...) @@ -77,6 +79,19 @@ struct neuron_device_bar_info { struct fw_io_bar_entry bars[NEURON_MAX_SWITCH_BARS]; }; +/** + * struct neuron_ham_throttle - per-device periodic HAM throttle state. + * + * @periodic_mc: slot-9 notification-queue host buffer per NC (allocated once, + * reused across resets). + * @agg_count: number of most-recent entries a sysfs read aggregates, per NC. + * 0 is treated as 1 (report only the latest entry). + */ +struct neuron_ham_throttle { + struct mem_chunk *periodic_mc[MAX_NC_PER_DEVICE]; + u32 agg_count[MAX_NC_PER_DEVICE]; +}; + struct neuron_device { struct pci_dev *pdev; int device_index; @@ -102,6 +117,9 @@ struct neuron_device { struct mem_chunk **nq_mc[MAX_NC_PER_DEVICE]; // memory chunk allocated for notification queue in each TOP_SP. struct mem_chunk **ts_nq_mc[MAX_TS_PER_DEVICE]; + // memory chunks allocated for notification queues on each H2D DMA engine, + // indexed by the dhal-provided dense H2D index (see ndhal_ndmar.h2d_eng_index). + struct mem_chunk *h2d_nq_mc[NUM_H2D_DMA_PER_DEVICE]; // memory chunk for setting device mem struct mem_chunk *memset_mc; @@ -114,8 +132,12 @@ struct neuron_device { struct neuron_metrics metrics; // aggregated metrics + struct neuron_fw_layout fw_layout; // reader for the firmware HBM layout + struct neuron_power power; // stats about device power utilization + struct neuron_ham_throttle ham_throttle; // periodic HAM throttle NQ state + struct neuron_crwl crwl[MAX_NC_PER_DEVICE]; // cooperative rw lock per NC. struct neuron_reset nr; // neuron reset thread's data @@ -145,3 +167,4 @@ struct neuron_device { }; #endif + diff --git a/neuron_dhal.c b/neuron_dhal.c index 179cb8e..986c73a 100644 --- a/neuron_dhal.c +++ b/neuron_dhal.c @@ -75,3 +75,4 @@ void neuron_dhal_free(void) kfree(ndhal); } } + diff --git a/neuron_dhal.h b/neuron_dhal.h index febfd03..d767d9c 100644 --- a/neuron_dhal.h +++ b/neuron_dhal.h @@ -4,6 +4,7 @@ #include "neuron_device.h" #include "neuron_fw_io.h" #include "neuron_mmap.h" +#include "neuron_platform.h" #include "neuron_sysfs_metrics.h" #if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) @@ -24,7 +25,7 @@ extern int force_die_flip; struct ndhal_arch { int arch; - enum neuron_platform_type platform_type; + const struct neuron_platform_desc *platform; u32 server_id; int (*narch_platform_ready) (struct neuron_device *nd, enum neuron_platform_operation_type platform_operation); }; @@ -49,7 +50,9 @@ struct ndhal_address_map { int seng_dma_eng_per_nd; int h2d_dma_eng_per_nd; int num_hbms; + u64 hbm_active_size; u16 nq_per_nc; + u32 supported_lnc_degrees; }; struct ndhal_reset { @@ -79,6 +82,16 @@ struct ndhal_nq { u8 (*nnq_get_nqid) (struct neuron_device *nd, u8 nc_id, u8 index, u32 nq_type); void (*nnq_set_hwaddr) (struct neuron_device *nd, u8 nc_id, u8 index, u32 nq_type, u32 size, u64 queue_pa); u8 (*nnq_get_nq_queue_count) (void); + + // h2d is shared between neuron cores so the nqids + // and nq functions need to be separate + int (*nnq_get_nq_h2d_id) (uint32_t h2d_eng_id); + uint32_t (*nnq_get_h2d_id) (int nq_h2d_id); + int (*nnq_set_h2d_hwaddr) (struct neuron_device *nd, uint32_t h2d_eng_id, u32 size, u64 queue_pa); + int (*nnq_set_nq_h2d_enable) (struct neuron_device *nd, uint32_t h2d_eng_id, u32 enable); + int (*nnq_get_nq_h2d_enable) (struct neuron_device *nd, uint32_t h2d_eng_id, u32 *enable); + // NQ_DEVICE_TYPE_H2D_DMA init takes the owning NC (NEURON_DRIVER_FEATURE_H2D_DMA_NQ_NC_ID) + bool h2d_dma_nq_nc_id_supported; }; struct ndhal_mpset { @@ -95,7 +108,7 @@ struct ndhal_ndmar { int (*ndmar_ctx_queue_bit) (uint32_t h2d_eng_id, uint32_t qid); void (*ndmar_ctx_queue_from_bit) (int bit, uint32_t *h2d_eng_id, uint32_t *qid); bool (*ndmar_is_h2t_def_q) (struct neuron_device *nd, uint32_t eng_id, uint32_t q_id); - bool (*nr_init_h2t_eng) ( int nc_idx, uint32_t nc_map); + bool (*nr_init_h2t_eng) ( int nc_idx, uint32_t nc_map); bool (*ndmar_is_nx_ring) (uint32_t eng_id, uint32_t q_id); int (*ndmar_quiesce_queues) (struct neuron_device *nd, u32 nc_id, u32 engine_count, u32 *queue_mask); }; @@ -120,13 +133,6 @@ struct ndhal_sysfs_metrics { char *arch_nc_type_suffix; char *arch_instance_suffix; char *arch_device_name_suffix; - int root_info_node_attrs_info_tbl_cnt; - nsysfsmetric_attr_info_t *root_info_node_attrs_info_tbl; - - int (*nsysfsmetric_add_ecc_nodes) (struct nsysfsmetric_metrics *metrics, - struct nsysfsmetric_node *stats_node, - int ecc_attrs_info_tbl_cnt, - const nsysfsmetric_attr_info_t *attr_info_tbl); void (*nsysfsmetric_get_hbm_error_count) (struct neuron_device *nd, bool repairable, @@ -147,6 +153,7 @@ struct ndhal_pci { int dram_bar; u64 dram_bar_size; + int (*neuron_pci_init) (struct neuron_device *nd); int (*neuron_pci_get_device_id) (struct neuron_device *nd, struct pci_dev *dev); int (*neuron_pci_device_id_to_rid_map) (uint32_t * count, uint32_t * did_to_rid_map); }; @@ -212,6 +219,12 @@ struct ndhal_perf { void (*perf_update_hbm_7200_supported) (struct neuron_device *nd); }; +struct ndhal_fw_metrics { + u32 max_misc_ram_version; // sent via SET_MAX_VERSION (vN/neuron_fw_metrics.h) + // decode one payload copied out by fw_layout_get_entry(); stub until schemas are adopted + int (*parse_payload) (struct neuron_device *nd, u16 type_id, const u8 *payload, u32 length); +}; + struct neuron_dhal { unsigned int pci_device_id; @@ -233,6 +246,7 @@ struct neuron_dhal { struct ndhal_npe ndhal_npe; struct ndhal_tpb ndhal_tpb; struct ndhal_perf ndhal_perf; + struct ndhal_fw_metrics ndhal_fw_metrics; void (*ndhal_ext_cleanup) (void); }; @@ -274,3 +288,4 @@ int ndhal_register_funcs_v3(void); int ndhal_register_funcs_v4(void); #endif // #ifndef NEURON_DHAL_H + diff --git a/neuron_dma.c b/neuron_dma.c index ff878d9..91025aa 100644 --- a/neuron_dma.c +++ b/neuron_dma.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -14,11 +15,11 @@ #include #include - #include "udma/udma.h" #include "neuron_trace.h" #include "neuron_device.h" #include "neuron_dma.h" +#include "neuron_dma_diag.h" #include "neuron_mempool.h" #include "neuron_mmap.h" #include "neuron_dhal.h" @@ -28,6 +29,7 @@ DECLARE_FAULT_ATTR(neuron_fail_dma_wait); #endif + int zerocopy_trn1_override = 0; module_param(zerocopy_trn1_override, int, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); MODULE_PARM_DESC(zerocopy_trn1_override, "override zerocopy for trn1"); @@ -67,14 +69,16 @@ struct neuron_pinned_mem_process { }; -static void ndma_ack_completed_desc(struct ndma_eng *eng, struct ndma_ring *ring, u32 count) +static void ndma_ack_completed_desc(struct ndma_eng *eng, struct ndma_ring *ring, u32 tx_count, u32 rx_count) { struct udma_q *rxq, *txq; udma_q_handle_get(&eng->udma, ring->qid, UDMA_TX, &txq); udma_q_handle_get(&eng->udma, ring->qid, UDMA_RX, &rxq); - udma_cdesc_ack(rxq, count); - udma_cdesc_ack(txq, count); + if (rx_count) + udma_cdesc_ack(rxq, rx_count); + if (tx_count) + udma_cdesc_ack(txq, tx_count); } u32 ndma_mc_pair_to_nc(struct mem_chunk *src_mc, struct mem_chunk *dst_mc) @@ -261,19 +265,17 @@ int ndma_memcpy_add_completion_desc( struct ndma_eng *eng, struct ndma_ring *rin return ret; } -int ndma_memcpy_wait_for_completion(struct ndma_eng *eng, struct ndma_ring *ring, u32 count, void * ptr, bool async, bool is_intra_device_dma) +int ndma_memcpy_wait_for_completion(struct ndma_eng *eng, struct ndma_ring *ring, u32 tx_count, u32 rx_count, void * ptr, bool async, bool is_intra_device_dma) { int ret = 0; volatile u32 *dst; volatile u32 *src; u64 i; u64 first_wait_time, wait; + /* wait time scales with descriptor count. Use the larger side so we don't underestimate. */ + u32 wait_count = (tx_count > rx_count) ? tx_count : rx_count; - ndhal->ndhal_ndma.ndma_get_wait_for_completion_time(count, async, &first_wait_time, &wait); - // Increase the wait time on virtual platforms - if (narch_is_qemu() || narch_is_emu()) { - wait = wait * 100 * 1000; - } + ndhal->ndhal_ndma.ndma_get_wait_for_completion_time(wait_count, async, &first_wait_time, &wait); if (is_intra_device_dma && !async) { first_wait_time = 10; // device-to-device DMA is much faster, just choose a small value independent of number of descriptors wait = wait/200; // can probably be set even lower if required @@ -305,13 +307,14 @@ int ndma_memcpy_wait_for_completion(struct ndma_eng *eng, struct ndma_ring *ring // for keeping track of which descriptors are free and can be allocated // Call ack in order to advance the counter, otherwise we eventually // run out of the descriptors to allocate on this ring - ndma_ack_completed_desc(eng, ring, count); + ndma_ack_completed_desc(eng, ring, tx_count, rx_count); break; } udelay(one_loop_sleep); } if (i > loop) { - pr_err("DMA completion timeout on nd%02d for %s q%d desc count %u\n", eng->nd->device_index, eng->udma.name, ring->qid, count); + NDMA_DIAG_DUMP_ENGINE(eng, ring, wait_count, "dma_completion_timeout"); + pr_err("DMA completion timeout on nd%02d for %s q%d tx desc count %u rx desc count %u\n", eng->nd->device_index, eng->udma.name, ring->qid, tx_count, rx_count); ret = -ETIMEDOUT; goto error; } @@ -334,33 +337,56 @@ int ndma_memcpy64k(struct ndma_eng *eng, struct ndma_ring *ring, dma_addr_t src, return ret; } + +// Legacy path outstanding sync transfer constraints +// Legacy sync/async IO path hold a lock for the duration of the transfer +// limiting the amount of outstanding requests possible, which prevents the +// ring from being overflowed. New contraints to coexist w/ zerocopy async flow. +// - max of 256 * 64K = 16MB per sync transfer + 1 completion descriptor. +// - max of 2 sync transfers outstanding + small pad (legacy async implementation) +// +#define NDMA_SYNC_XFER_THRESHOLD 256 +#define NDMA_SYNC_REQ_OUTSTANDING_MAX 2 +#define NDMA_SYNC_XFER_OUTSTANDING_MAX ((NDMA_SYNC_XFER_THRESHOLD+1) * NDMA_SYNC_REQ_OUTSTANDING_MAX + UDMA_MAX_NUM_CDESC_PER_CACHE_LINE) + +static_assert(NDMA_SYNC_XFER_OUTSTANDING_MAX < DMA_H2T_DESC_COUNT/4); + /** - * ndma_memcpy_chunks() + * ndma_pa_build_n_issue_descs() * + * Build and issue descriptors for a physical address to physical address + * transfer, append the completion descriptor and kick off the DMAs. * - * caveats/notes: - * need to figure out inuse & cleanup + * Shared by the legacy sync/async path (ndma_memcpy_chunks()) and the zerocopy + * async pipeline's PA memcpy request type, so barrier policy and chunking live + * in one place. * + * @src/@dst: base addresses of the transfer + * @size: bytes remaining to transfer + * @smove/@dmove: apply @offset to src / dst (see ndma_memcpy_offset_move()) + * @offset: offset into the transfer to start from + * @max_data_descs: cap on the number of data descriptors issued in this batch. + * Fewer bytes than @size are issued when the cap is hit. + * @completion_ptr: completion buffer the completion descriptor writes to + * @bytes_issued: bytes covered by the descriptors issued + * @nr_data_descs: number of data descriptors issued, excluding the completion descriptor + * @last: last request for this transfere */ -static int ndma_memcpy_chunks( struct ndma_eng *eng, struct ndma_ring *ring, struct ndma_h2t_dma_context * dma_ctx) +static int ndma_pa_build_n_issue_descs(struct ndma_eng *eng, struct ndma_ring *ring, + dma_addr_t src, dma_addr_t dst, u64 size, + bool smove, bool dmove, u64 offset, + u32 max_data_descs, void *completion_ptr, + u64 *bytes_issued, u32 *nr_data_descs, bool last) { int ret; - dma_addr_t src; - dma_addr_t dst; - u64 chunk_size; - u64 remaining; - u64 offset; - int pending_transfers; - bool done; - const u32 sync_threshold = DMA_H2T_DESC_COUNT/2 - UDMA_MAX_NUM_CDESC_PER_CACHE_LINE - 1; - - src = dma_ctx->src; - dst = dma_ctx->dst; - remaining = dma_ctx->remaining; - offset = dma_ctx->offset; - done = false; - pending_transfers = 0; - chunk_size = MAX_DMA_DESC_SIZE; + u64 chunk_size = MAX_DMA_DESC_SIZE; + u64 remaining = size; + u32 pending_transfers = 0; + bool done = false; + int barrier_type; + + *bytes_issued = 0; + *nr_data_descs = 0; while (!done) { dma_addr_t src_offset; @@ -368,38 +394,41 @@ static int ndma_memcpy_chunks( struct ndma_eng *eng, struct ndma_ring *ring, str if (remaining <= MAX_DMA_DESC_SIZE) { chunk_size = remaining; - } + } - if ((chunk_size == remaining) || (pending_transfers == sync_threshold)) { + if ((chunk_size == remaining) || (pending_transfers == max_data_descs - 1)) { done = true; } - src_offset = dma_ctx->smove ? src + offset : src; - dst_offset = dma_ctx->dmove ? dst + offset : dst; + src_offset = smove ? src + offset : src; + dst_offset = dmove ? dst + offset : dst; - ret = ndma_memcpy64k(eng, ring, src_offset, dst_offset, (u32)chunk_size, ndhal->ndhal_ndma.ndma_get_m2m_barrier_type(done)); - if (ret) { + ret = ndma_memcpy64k(eng, ring, src_offset, dst_offset, (u32)chunk_size, ndhal->ndhal_ndma.ndma_get_m2m_barrier_type(done && last)); + if (ret) { return ret; } - offset += chunk_size; + offset += chunk_size; remaining -= chunk_size; pending_transfers++; - + //TODO trace_dma_memcpy(nd, nc_id, src_offset, dst_offset, chunk_size, pending_transfers); } - // write completion descriptor, kick off DMAs, record pending xfers and data outstanding and prefetch if requested - // - ret = ndma_memcpy_add_completion_desc( eng, ring, dma_ctx->completion_ptr, UDMA_M2M_BARRIER_NONE); + *bytes_issued = size - remaining; + *nr_data_descs = pending_transfers; + + // write completion descriptor and kick off DMAs + // SOW isn't strictly necessary since if use use DMA on last, but it doesn't hurt anything and is needed if we + // don't use DMB in the future when we have an overlap checker + barrier_type = (last && (narch_get_arch() != NEURON_ARCH_V2)) ? UDMA_M2M_BARRIER_SOW : UDMA_M2M_BARRIER_NONE; + ret = ndma_memcpy_add_completion_desc( eng, ring, completion_ptr, barrier_type); if (ret) { - return ret; + return ret; } pending_transfers++; - dma_ctx->pending_transfers = pending_transfers; - dma_ctx->outstanding = dma_ctx->remaining - remaining; - + ret = udma_m2m_copy_start(&eng->udma, ring->qid, pending_transfers, pending_transfers); if (ret) { pr_err("failed to start DMA descriptor for %s q%d\n", eng->udma.name, ring->qid); @@ -409,6 +438,34 @@ static int ndma_memcpy_chunks( struct ndma_eng *eng, struct ndma_ring *ring, str return 0; } +/** + * ndma_memcpy_chunks() + * + * + * caveats/notes: + * need to figure out inuse & cleanup + * + */ +static int ndma_memcpy_chunks( struct ndma_eng *eng, struct ndma_ring *ring, struct ndma_h2t_dma_context * dma_ctx) +{ + int ret; + u64 bytes_issued; + u32 nr_data_descs; + + ret = ndma_pa_build_n_issue_descs(eng, ring, dma_ctx->src, dma_ctx->dst, dma_ctx->remaining, + dma_ctx->smove, dma_ctx->dmove, dma_ctx->offset, + NDMA_SYNC_XFER_THRESHOLD, dma_ctx->completion_ptr, + &bytes_issued, &nr_data_descs, true); + + // record pending xfers and data outstanding (the caller prefetches if requested) + /* ndma_memcpy_chunks emits paired m2s/s2m descriptors, so tx and rx counts match. */ + dma_ctx->pending_tx_transfers = nr_data_descs + 1; + dma_ctx->pending_rx_transfers = nr_data_descs + 1; + dma_ctx->outstanding = bytes_issued; + + return ret; +} + static int _ndma_memcpy_wait_for_completion( struct neuron_device *nd, u32 nc_id, int qid, struct ndma_eng *eng, struct ndma_ring *ring, struct ndma_h2t_dma_context * dma_ctx, struct ndma_h2t_dma_context * ndma_ctx) { @@ -417,7 +474,7 @@ static int _ndma_memcpy_wait_for_completion( struct neuron_device *nd, u32 nc_id while(true) { - ret = ndma_memcpy_wait_for_completion(eng, ring, dma_ctx->pending_transfers, dma_ctx->completion_ptr, async, false); + ret = ndma_memcpy_wait_for_completion(eng, ring, dma_ctx->pending_tx_transfers, dma_ctx->pending_rx_transfers, dma_ctx->completion_ptr, async, false); if (ret == 0) return ret; @@ -501,7 +558,8 @@ static int ndma_memcpy_offset_move(struct neuron_device *nd, u32 nc_id, dma_addr dma_ctx->dst = dst; dma_ctx->offset = 0ull; dma_ctx->remaining = size; - dma_ctx->pending_transfers = 0ull; + dma_ctx->pending_tx_transfers = 0; + dma_ctx->pending_rx_transfers = 0; dma_ctx->size = size; dma_ctx->smove = smove; dma_ctx->dmove = dmove; @@ -934,15 +992,6 @@ int ndma_bar0_blocked_one_engine(u64 base, u64 off) * Zero copy impementation. */ -/* Context for tracking a single tensor batch operation on submit flow */ -struct ndma_h2t_zcdma_op_context { - void *host_addr; - dma_addr_t dev_addr; - u64 offset; - u64 pin_size; - u64 remaining; -}; - /* DMA context state */ enum ndma_zcdma_state { NDMA_INVALID = 0, @@ -958,14 +1007,17 @@ struct ndma_h2t_zcdma_context { struct ndma_ring *ring; // ring // Submission-related - void *host_addr; // host address + void *host_addr; // host address (user VA; NULL for PA to PA transfers) + dma_addr_t src_pa; // source address for PA to PA transfers (NDMA_H2T_REQ_PA_MEMCPY); + // the destination is dev_addr and host_addr/page_list are unused dma_addr_t dev_addr; // device address u64 size; // size for this transfer bool direction; // direction. true = to-device/write/copy-in bool last; // last transfer for the entire request. u64 start_time; // start time for this transfer int nr_pages; // number of pages for this transfer - int nr_desc; // number of descriptors which is equal to pending transfers -1 + int nr_tx_desc; // number of tx (m2s) descriptors, excluding completion desc + int nr_rx_desc; // number of rx (s2m) descriptors, excluding completion desc struct page **page_list; // page structures tracking our pinned pages; // managed by page_list_pool in ctx queue enum ndma_zcdma_state state; // state of this transfer @@ -987,40 +1039,47 @@ struct ndma_h2t_zcdma_context { // used only for async BAR4 writes and temporary fake-async D2D copy }; -static void ndma_pinned_mem_process_release(struct kref *kref); - -static void ndma_zc_release_ctx(struct ndma_h2t_zcdma_context *ctx, u64 *nr_pinned_pages) -{ - // do not free or set completion_ptr null. it is managed by completion_pool in ctx queue - // do not free or set page_list null. it is managed by page_list_pool in ctx queue - - if (ctx->state >= NDMA_PINNED_UNSUBMITTED && ctx->nr_pages > 0) { - /* Only unpin if we pinned it ourselves (not pre-pinned memory) */ - if (!ctx->prepin_proc) { - if (ctx->direction) { - unpin_user_pages(ctx->page_list, ctx->nr_pages); - } else { - unpin_user_pages_dirty_lock(ctx->page_list, ctx->nr_pages, true); - } - } else { - kref_put(&ctx->prepin_proc->refcount, ndma_pinned_mem_process_release); - } - - *nr_pinned_pages -= ctx->nr_pages; - } - ctx->nr_pages = 0; - ctx->prepin_proc = NULL; - - if (ctx->mm) { - mmput(ctx->mm); - ctx->mm = NULL; - } +/* + * H2D DMA Context Queue + * --------------------- + * A fixed-size circular buffer storing pointers to `ndma_h2t_zcdma_context`. + * The queue tracks DMA contexts + * and maintains four logical indices: + * + * 1. head + * - Index of the first *valid* (non-empty) entry in the queue. + * - The context at `head` may be in any ndma_zcdma_state except + * NDMA_COMPLETED. + * + * 2. tail + * - Index of the next free slot where a new context pointer will be + * inserted. + * + * Queue state conditions: + * - Empty: head == tail + * - Full: head == ((tail + 1) & (capacity - 1)) + * - Current size: (tail - head + capacity) & (capacity - 1) + * + * @entries: Array of DMA context entries in the queue + * @completion_pool: Pre-allocated pool of completion_ptr buffer + * @page_list_pool: Pre-allocated pool of page_list arrays + * @capacity_mask: Maximum number of entries the queue can hold and minus one, + * capacity = capacity_mask + 1 + * @head: Index pointing to the head of the queue + * @tail: Index pointing to the tail of the queue + * @nr_pinned_pages: Total count of pinned memory pages belonged to the queue + */ +struct ndma_ctx_queue { + struct ndma_h2t_zcdma_context *entries; + void *completion_pool; + void *page_list_pool; + u32 capacity_mask; + u32 head; + u32 tail; + u64 nr_pinned_pages; +}; - ctx->state = NDMA_INVALID; - ctx->sequence_num = 0; - ctx->context = NULL; - ctx->compl_ret = 0; -} +static void ndma_pinned_mem_process_release(struct kref *kref); /* H2D DMA Completion Queue (CQ) */ #define NDMA_H2D_COMPL_QUEUE_CAPACITY 1024 @@ -1071,43 +1130,6 @@ void ndma_h2d_compl_queue_destroy(struct ndma_h2d_compl_queue *compl_queue) compl_queue->tail = 0; } -/* - * Emit the CQE for an accepted async io request. Caller must hold the ring - * lock that serializes compl_queue->tail. Submission failures return an error - * to userspace and must not call this function to emit a CQE. - */ -static void ndma_h2d_compl_queue_put(struct ndma_h2d_compl_queue *compl_queue, - u64 sequence_num, - s64 compl_ret, - void *context) -{ - u32 head = 0; - u32 tail = 0; - neuron_h2d_dma_compl_queue_t *compl_queue_shared = compl_queue->compl_queue_shared; - neuron_h2d_dma_compl_queue_entry_t *entry = NULL; - - head = smp_load_acquire(&compl_queue_shared->head); - tail = compl_queue->tail; - - while ((tail - head) >= (compl_queue->capacity_mask + 1)) { - pr_warn_once("h2d dma completion queue full; blocking until space is available\n"); - msleep(1); - head = smp_load_acquire(&compl_queue_shared->head); - tail = compl_queue->tail; - } - - entry = &compl_queue_shared->entries[tail & compl_queue->capacity_mask]; - - /* Write completion result to tail */ - entry->compl_ret = compl_ret; - entry->context = context; - entry->sequence_num = sequence_num; - - /* Move tail */ - compl_queue->tail = tail + 1; - smp_store_release(&compl_queue_shared->tail, compl_queue->tail); -} - #define NDMA_ZC_PAGES_PER_XFER 64 // number of pages in each zero copy dma transfer. This is somewhat, but not // totally arbitrary. We don't want to pin a lot of pages. We just want to // pin enough where (approximately): @@ -1117,22 +1139,18 @@ static void ndma_h2d_compl_queue_put(struct ndma_h2d_compl_queue *compl_queue, // Right now the sweet spot looks to be ~ 64 pages. More tuning is required. #define NDMA_ZC_MIN_PAGES_PER_XFER 64 -/* Hysteresis thresholds for descriptor wait checks in submission flow. */ -#define NDMA_ZC_DESC_WAIT_THRESHOLD_LO (NDMA_ZC_PAGES_PER_XFER + 1) -#define NDMA_ZC_DESC_WAIT_THRESHOLD_HI (NDMA_ZC_DESC_WAIT_THRESHOLD_LO * 8) +#define NDMA_ZC_PA_DESCS_PER_XFER 64 // number of descriptors in each physical address to physical address + // (NDMA_H2T_REQ_PA_MEMCPY) transfer step. Nothing is pinned on this path and + // both sides are contiguous, so the pipelining granularity is expressed in + // descriptors instead of pages. Same tradeoff as NDMA_ZC_PAGES_PER_XFER: + // large enough to keep the engine busy, small enough to interleave with + // other requests on the ring. +#define NDMA_ZC_PA_XFER_MAX ((u64)NDMA_ZC_PA_DESCS_PER_XFER * MAX_DMA_DESC_SIZE) /* DMA ctx queue constants */ #define NDMA_CTX_QUEUE_DEFAULT_CAPACITY 1024 #define NDMA_CTX_QUEUE_MAX_PINNED_PAGES 524288 -/* Skip tombstone ctxs */ -static void ndma_ctx_queue_advance_to_valid(struct ndma_ctx_queue *queue, u32 *idx, u32 stop) -{ - while (*idx != stop && queue->entries[*idx].state == NDMA_INVALID) { - *idx = (*idx + 1) & queue->capacity_mask; - } -} - /* Check empty or full */ static bool ndma_ctx_queue_is_empty(const struct ndma_ctx_queue *queue) { @@ -1144,66 +1162,9 @@ static bool ndma_ctx_queue_is_full(const struct ndma_ctx_queue *queue) return queue->head == ((queue->tail + 1) & queue->capacity_mask); } -static bool ndma_ctx_queue_submitted_empty(const struct ndma_ctx_queue *queue) -{ - return queue->head == queue->first_pinned_unsubmitted; -} - -static bool ndma_ctx_queue_pinned_unsubmitted_empty(const struct ndma_ctx_queue *queue) -{ - return queue->first_pinned_unsubmitted == queue->first_unpinned; -} - -static bool ndma_ctx_queue_unpinned_empty(const struct ndma_ctx_queue *queue) -{ - return queue->first_unpinned == queue->tail; -} - -/* Increment to next index */ -static void ndma_ctx_queue_inc_first_pinned_unsubmitted(struct ndma_ctx_queue *queue) -{ - if (ndma_ctx_queue_pinned_unsubmitted_empty(queue)) { - return; - } - queue->first_pinned_unsubmitted = (queue->first_pinned_unsubmitted + 1) & queue->capacity_mask; - ndma_ctx_queue_advance_to_valid(queue, &queue->first_pinned_unsubmitted, queue->first_unpinned); -} - -static void ndma_ctx_queue_inc_first_unpinned(struct ndma_ctx_queue *queue) -{ - if (ndma_ctx_queue_unpinned_empty(queue)) { - return; - } - queue->first_unpinned = (queue->first_unpinned + 1) & queue->capacity_mask; - ndma_ctx_queue_advance_to_valid(queue, &queue->first_unpinned, queue->tail); -} - static void ndma_ctx_queue_inc_tail(struct ndma_ctx_queue *queue) { - u32 old_tail = queue->tail; - u32 new_tail = (old_tail + 1) & queue->capacity_mask; - - // Assume the ctx at old tail is already filled by caller - // Tail advance may also initialize/advance the pinned+unsubmitted and unpinned pointers - struct ndma_h2t_zcdma_context *ctx = &queue->entries[old_tail]; - if (ctx->state == NDMA_PINNED_UNSUBMITTED || ctx->state == NDMA_COMPLETED) { - if (ndma_ctx_queue_pinned_unsubmitted_empty(queue)) { - // The first pinned+unsubmitted pointer appears at old_tail - queue->first_pinned_unsubmitted = old_tail; - } - if (ndma_ctx_queue_unpinned_empty(queue)) { - // No unpinned elements yet; start after the new tail - queue->first_unpinned = new_tail; - } - } else if (ctx->state == NDMA_UNPINNED) { - if (ndma_ctx_queue_unpinned_empty(queue)) { - // The first unpinned pointer appears at old_tail - queue->first_unpinned = old_tail; - } - } - - // Move tail forward after updating the two pointers - queue->tail = new_tail; + queue->tail = (queue->tail + 1) & queue->capacity_mask; } /* Peek */ @@ -1215,117 +1176,28 @@ static struct ndma_h2t_zcdma_context *ndma_ctx_queue_peek_tail(struct ndma_ctx_q return &queue->entries[queue->tail]; } -static struct ndma_h2t_zcdma_context *ndma_ctx_queue_peek_pinned_unsubmitted(struct ndma_ctx_queue *queue) -{ - if (ndma_ctx_queue_pinned_unsubmitted_empty(queue)) { - return NULL; - } - return &queue->entries[queue->first_pinned_unsubmitted]; -} - -static struct ndma_h2t_zcdma_context *ndma_ctx_queue_peek_first_unpinned(struct ndma_ctx_queue *queue) +static struct ndma_h2t_zcdma_context *ndma_ctx_queue_peek_head(struct ndma_ctx_queue *queue) { - if (ndma_ctx_queue_unpinned_empty(queue)) { + if (ndma_ctx_queue_is_empty(queue)) { return NULL; } - return &queue->entries[queue->first_unpinned]; + return &queue->entries[queue->head]; } /* Pop */ static struct ndma_h2t_zcdma_context *ndma_ctx_queue_pop_head(struct ndma_ctx_queue *queue) { - u32 old_head; - struct ndma_h2t_zcdma_context *ctx = NULL; - + struct ndma_h2t_zcdma_context *ctx; if (ndma_ctx_queue_is_empty(queue)) { return NULL; } - - old_head = queue->head; - ctx = &queue->entries[old_head]; + ctx = &queue->entries[queue->head]; queue->head = (queue->head + 1) & queue->capacity_mask; - ndma_ctx_queue_advance_to_valid(queue, &queue->head, queue->tail); - - if (ndma_ctx_queue_is_empty(queue)) { - queue->first_pinned_unsubmitted = queue->tail; - queue->first_unpinned = queue->tail; - } else { - if (old_head == queue->first_pinned_unsubmitted) { - ndma_ctx_queue_inc_first_pinned_unsubmitted(queue); - } - if (old_head == queue->first_unpinned) { - ndma_ctx_queue_inc_first_unpinned(queue); - } - } - return ctx; } -static struct ndma_h2t_zcdma_context *ndma_ctx_queue_pop_submitted(struct ndma_ctx_queue *queue) -{ - if (ndma_ctx_queue_submitted_empty(queue)) { - return NULL; - } - - return ndma_ctx_queue_pop_head(queue); -} - -/* Failure-path helper. - * Given a sequence number of a async request, wait for any matching submitted ctxs, then reset all matching ctxs. - * This prevents further remote pinning and submitting on a failed async request. - * Once a wait times out, skip waiting on subsequent contexts (engine is stuck). - */ -static void ndma_ctx_queue_drain_sequence(struct ndma_ctx_queue *queue, u64 sequence_num) -{ - u32 idx; - bool timed_out = false; - - for (idx = queue->head; idx != queue->tail; idx = (idx + 1) & queue->capacity_mask) { - struct ndma_h2t_zcdma_context *ctx = &queue->entries[idx]; - - if (ctx->sequence_num == sequence_num) { - // wait for already submitted DMAs to complete, unless engine already timed out. - if (ctx->state == NDMA_SUBMITTED && !timed_out) { - if (ndma_memcpy_wait_for_completion(ctx->eng, ctx->ring, ctx->nr_desc + 1, ctx->completion_ptr, false, false)) - timed_out = true; - } - - // release pinned pages and mm, and set state to invalid (tombstone). - ndma_zc_release_ctx(ctx, &queue->nr_pinned_pages); - } - } - - // After draining, advance the pointers to skip the invalidated ctxs. - ndma_ctx_queue_advance_to_valid(queue, &queue->first_unpinned, queue->tail); - ndma_ctx_queue_advance_to_valid(queue, &queue->first_pinned_unsubmitted, queue->first_unpinned); - ndma_ctx_queue_advance_to_valid(queue, &queue->head, queue->tail); -} - -/* Failure-path helper. - * Wait for submitted contexts from head up to (but not including) first_pinned_unsubmitted. - * Unpin from head up to (but not including) first_unpinned. - * Once a wait times out, skip waiting on subsequent contexts (engine is stuck). - */ -static void ndma_ctx_queue_drain(struct ndma_eng *eng, - struct ndma_ring *ring, - struct ndma_ctx_queue *queue) -{ - bool timed_out = false; - - while (!ndma_ctx_queue_is_empty(queue)) { - struct ndma_h2t_zcdma_context *ctx = ndma_ctx_queue_pop_head(queue); - - if (ctx->state == NDMA_SUBMITTED && !timed_out) { - if (ndma_memcpy_wait_for_completion(eng, ring, ctx->nr_desc + 1, ctx->completion_ptr, false, false)) - timed_out = true; - } - - ndma_zc_release_ctx(ctx, &queue->nr_pinned_pages); - } -} - /* Init and destroy queue */ -int ndma_ctx_queue_init(struct ndma_ctx_queue *queue) +static int ndma_ctx_queue_init(struct ndma_ctx_queue *queue) { int i; @@ -1344,8 +1216,6 @@ int ndma_ctx_queue_init(struct ndma_ctx_queue *queue) queue->capacity_mask = capacity - 1; queue->head = 0; queue->tail = 0; - queue->first_pinned_unsubmitted = 0; - queue->first_unpinned = 0; queue->entries = kvcalloc(capacity, sizeof(*queue->entries), GFP_KERNEL); if (!queue->entries) { @@ -1393,7 +1263,7 @@ int ndma_ctx_queue_init(struct ndma_ctx_queue *queue) return -ENOMEM; } -void ndma_ctx_queue_free(struct ndma_eng *eng, struct ndma_ring *ring, struct ndma_ctx_queue *queue) +static void ndma_ctx_queue_free(struct ndma_eng *eng, struct ndma_ring *ring, struct ndma_ctx_queue *queue) { int bit = ndhal->ndhal_ndmar.ndmar_ctx_queue_bit(eng->eng_id, ring->qid); @@ -1403,7 +1273,6 @@ void ndma_ctx_queue_free(struct ndma_eng *eng, struct ndma_ring *ring, struct nd return; } if (queue->entries) { - ndma_ctx_queue_drain(eng, ring, queue); kvfree(queue->entries); queue->entries = NULL; } @@ -1500,23 +1369,17 @@ static int ndma_build_n_issue_zc_descs(struct ndma_h2t_zcdma_context * dma_ctx) chunk_size = (remaining < contig_size) ? remaining : contig_size; if (chunk_size > MAX_DMA_DESC_SIZE) chunk_size = MAX_DMA_DESC_SIZE; - // on the read path completion write follows data writes in order, that means when the completion write finishes - // it's guaranteed that all the data has been written, no need for a barrier - - // on the write path we only need the barrier for the last transfer (the last set of pinned pages), why? - // HBM writes (data) and host write (completion) take different path through data fabric. That means w/o a barrier - // it's possible for the completion to be written before the data. - - // We don't need the barrier to ensure it's safe to unpin. - // s2m descriptors are executed in order, that means when s2m completion write is executed all s2m data writes - // have been executed as well, that means all m2s data reads have been executed, that means it's safe to unpin - - // use WRITE_BARRIER on V2 (set on the last data descriptor) - // use SOW on V3+ (set on completion descriptor below) - if (narch_get_arch() == NEURON_ARCH_V2) - barrier_type = (remaining == chunk_size && dma_ctx->direction && dma_ctx->last) ? UDMA_M2M_BARRIER_WRITE_BARRIER : UDMA_M2M_BARRIER_NONE; - else - barrier_type = UDMA_M2M_BARRIER_NONE; + // To avoid data races, we use DMB for both reads and writes to protect against overlaps. + // The old code which had lock protection around data transfers was less agressive and could assume the read + // path didn't need barriers because the completion cookie provided an implicit barrier. Async pipelining defeats + // that barrier. + // barrier_type = (remaining == chunk_size && dma_ctx->direction && dma_ctx->last) ? UDMA_M2M_BARRIER_DMB : UDMA_M2M_BARRIER_NONE; + // + // FWIW, We don't need the barrier to ensure it's safe to unpin. We rely on s2m write ordering to ensure pinned + // pages have no outstanding operations, which is why we use SOW on V3+ architectures for the completion descriptor. + // For device to host it guarantees all host data is written, for host to device, it also implicitly guaratees + // all m2s data reads are complete so we are safe to unpin. + barrier_type = (remaining == chunk_size && dma_ctx->last) ? UDMA_M2M_BARRIER_DMB : UDMA_M2M_BARRIER_NONE; ret = udma_m2m_copy_prepare_one(&dma_ctx->eng->udma, dma_ctx->ring->qid, src_addr, dst_addr, chunk_size, barrier_type, false); if (ret) { @@ -1532,8 +1395,12 @@ static int ndma_build_n_issue_zc_descs(struct ndma_h2t_zcdma_context * dma_ctx) } } - dma_ctx->nr_desc = pending_transfers; + /* zero-copy path emits paired m2s/s2m descriptors, so tx and rx counts match. */ + dma_ctx->nr_tx_desc = pending_transfers; + dma_ctx->nr_rx_desc = pending_transfers; + // completion descriptor needs a SOW (strong ordered writes) to push data out prior to completion landing + // This guarantess both the request are if (narch_get_arch() != NEURON_ARCH_V2) barrier_type = (dma_ctx->direction && dma_ctx->last) ? UDMA_M2M_BARRIER_SOW: UDMA_M2M_BARRIER_NONE; else @@ -1555,48 +1422,23 @@ static int ndma_build_n_issue_zc_descs(struct ndma_h2t_zcdma_context * dma_ctx) return ret; } -/* Return the number of descriptors available (TX-only; TX/RX counts match) */ -static u32 ndma_zc_descs_available(struct ndma_eng *eng, u32 qid) +/* + * Return TX and RX descriptor availability for the paired m2m ring. + * + * Both counts are populated in a single pair of queue-handle lookups. + * Callers can compare each side against its own required count, so this + * supports asymmetric submissions where tx_count != rx_count. + */ +static void ndma_zc_descs_available(struct ndma_eng *eng, u32 qid, u32 *tx_avail_out, u32 *rx_avail_out) { struct udma_q *txq; - u32 tx_desc_available; + struct udma_q *rxq; udma_q_handle_get(&eng->udma, qid, UDMA_TX, &txq); + udma_q_handle_get(&eng->udma, qid, UDMA_RX, &rxq); - tx_desc_available = udma_available_get(txq); - - /* TX/RX descriptor availability is kept in lock-step. */ - return tx_desc_available; -} - -/* Estimate if a zero-copy DMA context fits in the available descriptors. */ -static bool _ndma_zc_descs_available(struct ndma_eng *eng, u32 qid, u32 threshold) -{ - u32 max_descs_required = threshold + 1; /* +1 for completion descriptor */ - - return ndma_zc_descs_available(eng, qid) >= max_descs_required; -} - -/* Whether we should wait for some completions before submitting more in the next iteration */ -static bool ndma_zc_should_wait(struct ndma_eng *eng, - struct ndma_ring *ring, - struct ndma_ctx_queue *ctx_queue, - u32 *desc_threshold) -{ - bool pinned_at_max; - bool desc_ring_full; - bool ctx_queue_full; - - pinned_at_max = ctx_queue->nr_pinned_pages >= NDMA_CTX_QUEUE_MAX_PINNED_PAGES; - ctx_queue_full = ndma_ctx_queue_is_full(ctx_queue); - desc_ring_full = !_ndma_zc_descs_available(eng, ring->qid, *desc_threshold); - - if (pinned_at_max || desc_ring_full || ctx_queue_full) { - *desc_threshold = NDMA_ZC_DESC_WAIT_THRESHOLD_HI; - return true; - } - - return false; + *tx_avail_out = udma_available_get(txq); + *rx_avail_out = udma_available_get(rxq); } static bool ndma_pinned_mem_try_populate(pid_t pid, u64 va, u64 size, struct page **page_list, int nr_pages, struct neuron_pinned_mem_process **prepin_proc); @@ -1624,6 +1466,11 @@ static int ndma_zerocopy_pin_pages(int nd_id, pr_err("remote pin requested without mm context\n"); return -EINVAL; } + /* pin_user_pages_remote() requires the caller to hold mmap_read_lock; + * unlike pin_user_pages()/pin_user_pages_fast() it does not take it + * internally. We run from the completion kthread (current->mm == NULL) + * against a stashed mm, so lock it explicitly here. */ + mmap_read_lock(dma_ctx->mm); #if (!defined(RHEL_RELEASE_CODE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0))) || (defined(RHEL_RELEASE_CODE) && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(9, 6))) nr_pinned = pin_user_pages_remote(dma_ctx->mm, (unsigned long)dma_ctx->host_addr & PAGE_MASK, @@ -1640,7 +1487,7 @@ static int ndma_zerocopy_pin_pages(int nd_id, NULL, NULL); #endif - mmput(dma_ctx->mm); + mmap_read_unlock(dma_ctx->mm); dma_ctx->mm = NULL; } else { nr_pinned = pin_user_pages_fast((unsigned long)dma_ctx->host_addr & PAGE_MASK, dma_ctx->nr_pages, @@ -1678,452 +1525,84 @@ static int ndma_zerocopy_pin_pages(int nd_id, return 0; } -int ndma_zerocopy_submit(struct neuron_device *nd, - u32 nc_id, - const nrt_tensor_batch_op_t *ops, - u32 num_ops, - dma_addr_t dev_base, - int qid, - bool direction, - u64 sequence_num, - void *context) +//forward +static int ndma_zca_process_ring_cmpltns(struct neuron_device *nd); + +static int ndma_h2d_cmpltn_thread_fn(void *arg) { + struct neuron_device *nd = (struct neuron_device *)arg; int ret = 0; - int i = 0; - const int eng_id = ndhal->ndhal_ndmar.ndmar_get_h2t_eng_id(nd, nc_id); - struct ndma_eng *eng = &nd->ndma_engine[eng_id]; - struct ndma_queue *queue = &eng->queues[qid]; - struct ndma_ring *ring = &queue->ring_info; - struct ndma_ctx_queue *ctx_queue = &ring->dma_ctx_queue; - struct ndma_h2t_zcdma_context *cur_ctx = NULL; - struct ndma_h2t_zcdma_op_context op_ctx; - bool async = (sequence_num != 0); - - /* Verify ring ownership. */ - if (!ndmar_h2t_ring_is_owner(ring, nc_id)) { - pr_err("nd%02d: attempting to use qid %d that was not assigned to nc %d\n", - nd->device_index, qid, nc_id); - return -ENOENT; - } - if (async) { - ret = ndma_h2d_create_cmpltn_thread(nd); - if (ret) { - return ret; + while (!kthread_should_stop() && !nd->dma_cmpltn_thread.stop) { + wait_event_interruptible(nd->dma_cmpltn_thread.wait_queue, + nd->dma_cmpltn_thread.stop || atomic64_read(&nd->dma_cmpltn_thread.nonempty_ctxq_bitmap) != 0); + if (kthread_should_stop() || nd->dma_cmpltn_thread.stop) { + break; } + ndma_zca_process_ring_cmpltns(nd); } + return ret; +} - mutex_lock(&ring->h2t_ring_lock); +int ndma_h2d_create_cmpltn_thread(struct neuron_device *nd) +{ + int ret = 0; + struct task_struct *thread; - for (i = 0; i < num_ops; i++) { - const nrt_tensor_batch_op_t *op = &ops[i]; - op_ctx.host_addr = op->buffer; - op_ctx.dev_addr = dev_base + op->offset; - op_ctx.offset = (unsigned long)op_ctx.host_addr & (PAGE_SIZE - 1); - op_ctx.remaining = op->size; - /* pin_size is in page units; include the page offset. */ - op_ctx.pin_size = ndma_calc_zc_pin_size(op_ctx.remaining + op_ctx.offset); - - while (op_ctx.remaining) { - int nr_pages; - bool can_pin; - bool ctx_queue_full; - - /* Step 1: submit any pinned contexts that have available descriptors. */ - while (true) { - struct ndma_h2t_zcdma_context *unsubmitted_ctx = ndma_ctx_queue_peek_pinned_unsubmitted(ctx_queue); - - if (!unsubmitted_ctx || !_ndma_zc_descs_available(eng, ring->qid, unsubmitted_ctx->nr_pages)) { - break; - } + if (READ_ONCE(nd->dma_cmpltn_thread.thread)) { + return 0; + } - ret = ndma_build_n_issue_zc_descs(unsubmitted_ctx); - if (ret) { - pr_err("failed to build and issue zero-copy descs\n"); - goto done; - } + mutex_lock(&nd->lock); - ndma_ctx_queue_inc_first_pinned_unsubmitted(ctx_queue); - } + if (nd->dma_cmpltn_thread.thread) { + /* thread already created */ + goto out; + } - /* Step 2: set up the current ctx if there is room and pinned-page budget. */ - nr_pages = DIV_ROUND_UP(op_ctx.pin_size, PAGE_SIZE); - can_pin = (ctx_queue->nr_pinned_pages + nr_pages <= NDMA_CTX_QUEUE_MAX_PINNED_PAGES); - ctx_queue_full = ndma_ctx_queue_is_full(ctx_queue); + nd->dma_cmpltn_thread.stop = false; + init_waitqueue_head(&nd->dma_cmpltn_thread.wait_queue); + atomic64_set(&nd->dma_cmpltn_thread.nonempty_ctxq_bitmap, 0); + thread = kthread_run(ndma_h2d_cmpltn_thread_fn, nd, "nd%d dma cmpltn", nd->device_index); + if (IS_ERR(thread)) { + ret = PTR_ERR(thread); + pr_err("h2d dma completion thread creation failed\n"); + goto out; + } + WRITE_ONCE(nd->dma_cmpltn_thread.thread, thread); - if (async && ctx_queue_full) { - pr_err("ctx queue full. failed to submit async ctx\n"); - ret = -EBUSY; - goto done; - } +out: + mutex_unlock(&nd->lock); + return ret; +} - if ((can_pin || async) && !ctx_queue_full) { - cur_ctx = ndma_ctx_queue_peek_tail(ctx_queue); - cur_ctx->eng = eng; - cur_ctx->ring = ring; - cur_ctx->host_addr = op_ctx.host_addr; - cur_ctx->dev_addr = op_ctx.dev_addr; - // First chunk may be unaligned; later chunks are page-aligned with offset=0. - cur_ctx->size = op_ctx.pin_size - op_ctx.offset; - cur_ctx->direction = direction; - cur_ctx->last = (cur_ctx->size == op_ctx.remaining && i == num_ops - 1); - cur_ctx->nr_pages = nr_pages; - cur_ctx->state = NDMA_UNPINNED; - cur_ctx->nr_desc = 0; // Set by ndma_build_n_issue_zc_descs(). - cur_ctx->mm = NULL; - cur_ctx->pid = task_tgid_nr(current); - cur_ctx->sequence_num = sequence_num; - cur_ctx->context = context; - - /* Pin now if possible; otherwise capture mm for remote pinning (async only). */ - if (can_pin) { - ret = ndma_zerocopy_pin_pages(nd->device_index, nc_id, ctx_queue, cur_ctx, false); - if (ret) { - pr_err("failed to pin pages for zero copy dma on nd %d\n", nd->device_index); - goto done; - } - } else if (async) { - struct mm_struct *mm = current->mm; - mmget(mm); - cur_ctx->mm = mm; - } +void ndma_h2d_stop_cmpltn_thread(struct neuron_device *nd) +{ + if (!nd->dma_cmpltn_thread.thread) { + return; + } + if (IS_ERR(nd->dma_cmpltn_thread.thread)) { + nd->dma_cmpltn_thread.thread = NULL; + return; + } - /* Advance the queue tail. - * May also initialize/advance the pinned+unsubmitted and unpinned pointers. - */ - ndma_ctx_queue_inc_tail(ctx_queue); - - /* Update loop variables for the next chunk. */ - op_ctx.remaining -= cur_ctx->size; - op_ctx.host_addr += cur_ctx->size; - op_ctx.dev_addr += cur_ctx->size; - op_ctx.pin_size = (op_ctx.remaining < op_ctx.pin_size) ? op_ctx.remaining : op_ctx.pin_size; - op_ctx.offset = 0; - cur_ctx = NULL; - } + nd->dma_cmpltn_thread.stop = true; + wake_up(&nd->dma_cmpltn_thread.wait_queue); + kthread_stop(nd->dma_cmpltn_thread.thread); + nd->dma_cmpltn_thread.thread = NULL; +} - /* Step 3 (sync): wait for submitted transfers to complete from the head. */ - if (!async) { - u32 desc_threshold = NDMA_ZC_DESC_WAIT_THRESHOLD_LO; +/* + * Pre-pinned host memory implementation + * Uses a global hash table keyed by PID, with each process having its own + * rbtree of pinned memory regions keyed by VA. + * Host memory is not device-specific — a process can pin via any device + * and the zerocopy path on any device will find the pre-pinned region. + */ - while (ndma_zc_should_wait(eng, ring, ctx_queue, &desc_threshold)) { - struct ndma_h2t_zcdma_context *submitted_ctx = ndma_ctx_queue_pop_submitted(ctx_queue); - - ret = ndma_memcpy_wait_for_completion(eng, ring, submitted_ctx->nr_desc + 1, - submitted_ctx->completion_ptr, - true, false); - ndma_zc_release_ctx(submitted_ctx, &ctx_queue->nr_pinned_pages); - if (ret) { - pr_err("failed to wait for completion of zero copy dma\n"); - goto done; - } - } - } - } - } - - if (!async) { - /* Step 4 (sync): submit remaining pinned ctxs, then drain all submitted ctxs. */ - while (true) { - struct ndma_h2t_zcdma_context *ctx_to_submit = ndma_ctx_queue_peek_pinned_unsubmitted(ctx_queue); - - if (ctx_to_submit && _ndma_zc_descs_available(eng, ring->qid, ctx_to_submit->nr_pages)) { - ret = ndma_build_n_issue_zc_descs(ctx_to_submit); - if (ret) { - pr_err("failed to build and issue zero-copy descs\n"); - goto done; - } - ndma_ctx_queue_inc_first_pinned_unsubmitted(ctx_queue); - } - - struct ndma_h2t_zcdma_context *ctx_to_wait = ndma_ctx_queue_pop_submitted(ctx_queue); - if (ctx_to_wait) { - ret = ndma_memcpy_wait_for_completion(eng, ring, ctx_to_wait->nr_desc + 1, - ctx_to_wait->completion_ptr, - true, false); - ndma_zc_release_ctx(ctx_to_wait, &ctx_queue->nr_pinned_pages); - if (ret) { - pr_err("failed to wait for completion of zero copy dma\n"); - goto done; - } - } - - if (!ctx_to_submit && !ctx_to_wait) { - break; - } - } - } - -done: - if (ret) { - ndma_ctx_queue_drain(eng, ring, ctx_queue); - } - - mutex_unlock(&ring->h2t_ring_lock); - - if (!ret && async) { - int bit = ndhal->ndhal_ndmar.ndmar_ctx_queue_bit(eng_id, qid); - atomic64_or(BIT_ULL(bit), &nd->dma_cmpltn_thread.nonempty_ctxq_bitmap); // set the bit for this queue - wake_up(&nd->dma_cmpltn_thread.wait_queue); - } - - return ret; -} - -int ndma_zerocopy_submit_completed(struct neuron_device *nd, u32 nc_id, int qid, - u64 sequence_num, s64 compl_ret, void *context) -{ - const int eng_id = ndhal->ndhal_ndmar.ndmar_get_h2t_eng_id(nd, nc_id); - struct ndma_eng *eng = &nd->ndma_engine[eng_id]; - struct ndma_ring *ring = &eng->queues[qid].ring_info; - struct ndma_ctx_queue *ctx_queue = &ring->dma_ctx_queue; - struct ndma_h2t_zcdma_context *ctx; - int ret; - - if (!ndmar_h2t_ring_is_owner(ring, nc_id)) { - pr_err("nd%02d: attempting to use qid %d that was not assigned to nc %d\n", - nd->device_index, qid, nc_id); - return -ENOENT; - } - - ret = ndma_h2d_create_cmpltn_thread(nd); - if (ret) { - return ret; - } - - mutex_lock(&ring->h2t_ring_lock); - if (ndma_ctx_queue_is_full(ctx_queue)) { - mutex_unlock(&ring->h2t_ring_lock); - return -EBUSY; - } - - ctx = ndma_ctx_queue_peek_tail(ctx_queue); - ctx->eng = eng; - ctx->ring = ring; - ctx->direction = true; - ctx->last = true; - ctx->nr_pages = 0; - ctx->nr_desc = 0; - ctx->state = NDMA_COMPLETED; - ctx->sequence_num = sequence_num; - ctx->context = context; - ctx->compl_ret = compl_ret; - ndma_ctx_queue_inc_tail(ctx_queue); - mutex_unlock(&ring->h2t_ring_lock); - - atomic64_or(BIT_ULL(ndhal->ndhal_ndmar.ndmar_ctx_queue_bit(eng_id, qid)), - &nd->dma_cmpltn_thread.nonempty_ctxq_bitmap); - wake_up(&nd->dma_cmpltn_thread.wait_queue); - return 0; -} - -/* The completion flow for completion, remote pinning, and submission. Async IO only */ -static int ndma_zerocopy_complete(struct neuron_device *nd, - struct ndma_eng *eng, - struct ndma_ring *ring, - u64 *nonempty_ctxq_bitmap_copy) -{ - int ret = 0; - int err = 0; - bool did_work = false; - struct ndma_ctx_queue *ctx_queue = NULL; - u32 desc_threshold = NDMA_ZC_DESC_WAIT_THRESHOLD_LO; - - if (!ring) { - return -EINVAL; - } - - ctx_queue = &ring->dma_ctx_queue; - - mutex_lock(&ring->h2t_ring_lock); - - /* 1) Wait for at least one submitted context to complete */ - while (true) { - if (ndma_ctx_queue_submitted_empty(ctx_queue)) { - break; - } - /* - * Async completion must always retire at least one submitted context. - * Only fall back to the wait-throttling heuristic after we have made - * some forward progress in this pass. - */ - if (did_work && !ndma_zc_should_wait(eng, ring, ctx_queue, &desc_threshold)) { - break; - } - struct ndma_h2t_zcdma_context *submitted_ctx = ndma_ctx_queue_pop_submitted(ctx_queue); - - if (submitted_ctx->state == NDMA_COMPLETED) { - // Dummy context: BAR4/fake-async copy completed before enqueue. - ret = submitted_ctx->compl_ret; - } else { - ret = ndma_memcpy_wait_for_completion(eng, ring, submitted_ctx->nr_desc + 1, - submitted_ctx->completion_ptr, true, false); - } - if (ret) { - err = ret; - pr_err("async h2d dma completion failed for seq num %llu: %d\n", submitted_ctx->sequence_num, ret); - ndma_h2d_compl_queue_put(&ring->dma_compl_queue, submitted_ctx->sequence_num, ret, submitted_ctx->context); - ndma_ctx_queue_drain_sequence(ctx_queue, submitted_ctx->sequence_num); - } else if (submitted_ctx->last) { - ndma_h2d_compl_queue_put(&ring->dma_compl_queue, submitted_ctx->sequence_num, 0, submitted_ctx->context); - } - - ndma_zc_release_ctx(submitted_ctx, &ctx_queue->nr_pinned_pages); - did_work = true; - } - - /* 2) Submit pinned but unsubmitted contexts */ - while (true) { - struct ndma_h2t_zcdma_context *pinned_unsubmitted_ctx = ndma_ctx_queue_peek_pinned_unsubmitted(ctx_queue); - - if (!pinned_unsubmitted_ctx) { - break; - } - if (pinned_unsubmitted_ctx->state == NDMA_COMPLETED) { - // Dummy context has no DMA descriptors; preserve CQ order only. - ndma_ctx_queue_inc_first_pinned_unsubmitted(ctx_queue); - did_work = true; - continue; - } - if (!_ndma_zc_descs_available(eng, ring->qid, pinned_unsubmitted_ctx->nr_pages)) { - break; - } - - ret = ndma_build_n_issue_zc_descs(pinned_unsubmitted_ctx); - if (ret) { - err = ret; - pr_err("async h2d dma submission failed for seq num %llu: %d\n", pinned_unsubmitted_ctx->sequence_num, ret); - ndma_h2d_compl_queue_put(&ring->dma_compl_queue, pinned_unsubmitted_ctx->sequence_num, ret, pinned_unsubmitted_ctx->context); - ndma_ctx_queue_drain_sequence(ctx_queue, pinned_unsubmitted_ctx->sequence_num); - } else { - ndma_ctx_queue_inc_first_pinned_unsubmitted(ctx_queue); - } - did_work = true; - } - - /* 3) Remote pin unpinned contexts */ - while (true) { - struct ndma_h2t_zcdma_context *unpinned_ctx = ndma_ctx_queue_peek_first_unpinned(ctx_queue); - - if (!unpinned_ctx || ctx_queue->nr_pinned_pages + unpinned_ctx->nr_pages > NDMA_CTX_QUEUE_MAX_PINNED_PAGES) { - break; - } - - ret = ndma_zerocopy_pin_pages(nd->device_index, ring->h2t_nc_id, ctx_queue, unpinned_ctx, true); - if (ret) { - err = ret; - pr_err("async h2d dma remote pinning failed for seq num %llu: %d\n", unpinned_ctx->sequence_num, ret); - ndma_h2d_compl_queue_put(&ring->dma_compl_queue, unpinned_ctx->sequence_num, ret, unpinned_ctx->context); - ndma_ctx_queue_drain_sequence(ctx_queue, unpinned_ctx->sequence_num); - } else { - ndma_ctx_queue_inc_first_unpinned(ctx_queue); - } - did_work = true; - } - - mutex_unlock(&ring->h2t_ring_lock); - - if (ndma_ctx_queue_is_empty(ctx_queue)) { - int bit = ndhal->ndhal_ndmar.ndmar_ctx_queue_bit(eng->eng_id, ring->qid); - *nonempty_ctxq_bitmap_copy &= ~BIT_ULL(bit); - } - - return err; -} - -static int ndma_h2d_cmpltn_thread_fn(void *arg) -{ - struct neuron_device *nd = (struct neuron_device *)arg; - int ret = 0; - - while (!kthread_should_stop() && !nd->dma_cmpltn_thread.stop) { - wait_event_interruptible(nd->dma_cmpltn_thread.wait_queue, - nd->dma_cmpltn_thread.stop || atomic64_read(&nd->dma_cmpltn_thread.nonempty_ctxq_bitmap) != 0); - if (kthread_should_stop() || nd->dma_cmpltn_thread.stop) { - break; - } - u64 bitmap = atomic64_xchg(&nd->dma_cmpltn_thread.nonempty_ctxq_bitmap, 0); - - while (bitmap) { - int bit = __ffs64(bitmap); - u32 eng_id; - u32 qid; - struct ndma_eng *eng; - struct ndma_ring *ring; - - ndhal->ndhal_ndmar.ndmar_ctx_queue_from_bit(bit, &eng_id, &qid); - - eng = &nd->ndma_engine[eng_id]; - ring = &eng->queues[qid].ring_info; - ret = ndma_zerocopy_complete(nd, eng, ring, &bitmap); - if (ret) { - pr_err("dma completion thread failed to process ctx queue for eng %d q %d: %d\n", eng_id, qid, ret); - } - } - } - - return ret; -} - -int ndma_h2d_create_cmpltn_thread(struct neuron_device *nd) -{ - int ret = 0; - struct task_struct *thread; - - if (READ_ONCE(nd->dma_cmpltn_thread.thread)) { - return 0; - } - - mutex_lock(&nd->lock); - - if (nd->dma_cmpltn_thread.thread) { - /* thread already created */ - goto out; - } - - nd->dma_cmpltn_thread.stop = false; - init_waitqueue_head(&nd->dma_cmpltn_thread.wait_queue); - atomic64_set(&nd->dma_cmpltn_thread.nonempty_ctxq_bitmap, 0); - thread = kthread_run(ndma_h2d_cmpltn_thread_fn, nd, "neuron dma cmpltn"); - if (IS_ERR(thread)) { - ret = PTR_ERR(thread); - pr_err("h2d dma completion thread creation failed\n"); - goto out; - } - WRITE_ONCE(nd->dma_cmpltn_thread.thread, thread); - -out: - mutex_unlock(&nd->lock); - return ret; -} - -void ndma_h2d_stop_cmpltn_thread(struct neuron_device *nd) -{ - if (!nd->dma_cmpltn_thread.thread) { - return; - } - if (IS_ERR(nd->dma_cmpltn_thread.thread)) { - nd->dma_cmpltn_thread.thread = NULL; - return; - } - - nd->dma_cmpltn_thread.stop = true; - wake_up(&nd->dma_cmpltn_thread.wait_queue); - kthread_stop(nd->dma_cmpltn_thread.thread); - nd->dma_cmpltn_thread.thread = NULL; -} - -/* - * Pre-pinned host memory implementation - * Uses a global hash table keyed by PID, with each process having its own - * rbtree of pinned memory regions keyed by VA. - * Host memory is not device-specific — a process can pin via any device - * and the zerocopy path on any device will find the pre-pinned region. - */ - -/* 256 buckets: up to 16 devices × 16 processes per device */ -static DEFINE_HASHTABLE(pinned_mem_htable, 8); -static DEFINE_MUTEX(pinned_mem_htable_lock); /* protects hash table add/remove/lookup only */ +/* 256 buckets: up to 16 devices × 16 processes per device */ +static DEFINE_HASHTABLE(pinned_mem_htable, 8); +static DEFINE_MUTEX(pinned_mem_htable_lock); /* protects hash table add/remove/lookup only */ /* * Find or create per-process state and take a reference. @@ -2493,3 +1972,1549 @@ void ndma_pinned_mem_cleanup_process(pid_t pid) kref_put(&proc->refcount, ndma_pinned_mem_process_release); } } + +/* + * NEW CODE to handle queuing of h2t DMA requests (sync and async) + * + * ndma_h2t_dma_ioq - contains h2t DMA related structures (except for completion queue atm) + * request queue - tracks user submitted requests + * context queue - tracks in flight data from requests. These use the existing context queue entries. + * need to be stripped down to minimal entries + * spill record queue - when completion queue is full, we place completions in the spill record queue + * + * So flow is: + * 1. add to request queue. + * 2. process requests into one or more context queue entries that track outstanding DMA operations. + * limited by credits as to how much we can process at a given go at it. + * set the service queue bit and kick the completion thread. + * 3. completion thread + * 3a. service rings (no-wait to see if there DMAs that have finished and process the + * associated context queue entris. + * 3b. service the spill record queue (to push held up async completions to the user) + * 3c. service the request queue. lock held. If idle, clear the service queue bit. unlock. + * + */ + +/* mimimized version of a context entry. not used yet + * + */ +struct ndma_h2t_zcdma_context_minimal { + dma_addr_t dev_addr; // device address (debug) + void *completion_ptr; // completion buffer pointer; + // host memory buffer which driver polls on for completions; + // managed by completion_pool in ctx queue + u64 sequence_num; // async sequence number; 0 for sync transfers + void *context; // async completion context + bool direction; // direction. true = to-device/write/copy-in + bool last; // last transfer for the entire request. + enum ndma_zcdma_state state; // state of this transfer (shrink to u16) + u16 nr_pages; // number of pages for this transfer (0 for bar4 writes) + u16 nr_tx_desc; // number of tx (m2s) descriptors, excluding completion desc + u16 nr_rx_desc; // number of rx (s2m) descriptors, excluding completion desc + union { + struct page **page_list; // page structures tracking our pinned pages; + s64 compl_ret; // result for in-order CQ completion of completed work. Only valid for async. + }; +}; + +/* what kind of transfer the queued request represents */ +enum ndma_h2t_req_type { + NDMA_H2T_REQ_DMA_MEMCPY = 0, // standard zerocopy DMA (pin pages, build descs, submit) + NDMA_H2T_REQ_BAR4_WR, // BAR4 mmio write - already completed, only enqueued for CQE ordering + NDMA_H2T_REQ_DESC_EXECUTE, // pre-built tx/rx descriptors supplied by userspace + NDMA_H2T_REQ_PA_MEMCPY, // physical address to physical address copy (mem chunk to mem chunk). + // no pages to pin, addresses are already DMA addresses +}; + +/** + * struct to track a queued request. Common bookkeeping is at the top; per-type + * fields live in the union below. `ops[]` is the trailing flexible array used + * only by NDMA_H2T_REQ_DMA_MEMCPY (kzalloc'd with num_ops * sizeof(op) tail). + */ +struct ndma_h2t_queued_request { + struct list_head queuelist; // linked list of waiting requests + enum ndma_h2t_req_type type; // request kind - selects processing branch + u64 sequence_num; // + void *context; // + pid_t pid; // PID of the process that initiated the copy + union { + struct { + u32 num_ops; + dma_addr_t dev_base; + bool direction; + struct mm_struct *mm; // mm that owns the user buffers + nrt_tensor_batch_op_t ops[]; // NDMA_H2T_REQ_DMA_MEMCPY only + } memcpy; + struct { + s64 compl_ret; // pre-computed BAR4 write status echoed in CQE + } bar4_wr; + struct { + dma_addr_t src_base; // source dma address + dma_addr_t dst_base; // destination dma address + u64 size; // NDMA_H2T_REQ_PA_MEMCPY only + } pa_memcpy; + struct { + union udma_desc *tx_desc_buf; // pointer to `descs` for the tx portion of the descriptors + union udma_desc *rx_desc_buf; // pointer to `descs` for the rx portion of the descriptors + u32 nr_tx_desc; + u32 nr_rx_desc; + union udma_desc descs[]; // kernel-side copy of user tx and rx descriptors + } desc_execute; + }; +}; + +/** + * if the h2t dma completion queue used to communicate to user space is full (application hasn't been consuming), + * we collect the overflow into a linked list of spill records that each have a smallish queue of completion + * queue entries. The records are compact to fit into 4k of memory and have a little over 300 entries in each. + * + */ +#define NDMA_H2T_COMPL_SPILL_REC_Q_CNT ((4096 - sizeof(struct ndma_h2t_compl_spill_q_rec_hdr)) / sizeof(neuron_h2d_dma_compl_queue_entry_t)) + +struct ndma_h2t_compl_spill_q_rec { + struct ndma_h2t_compl_spill_q_rec_hdr { + struct list_head queuelist; // linked list of spill queue recs + u16 head; // head in this spill q + u16 tail; // tail in this spill q + } hdr; + neuron_h2d_dma_compl_queue_entry_t queue[NDMA_H2T_COMPL_SPILL_REC_Q_CNT]; +}; + +/** + * h2t dma ioq structure + * + * need to describe purpose + */ +struct ndma_h2t_dma_ioq { + struct ndma_eng *eng; + struct ndma_ring *ring; + u32 qid; + struct list_head queued_req_head; + u64 pin_retries; + struct ndma_ctx_queue ctx_q; + u64 compl_timeout_deadline; + bool cleanup; + + // execution context of current op for (head) request being processed + // only relevant for requests that are managing a data transfer. Other + // requests are handled immediatly (CCE/Bar4 for example are handled immediately) + struct ndma_h2t_queued_req_op_ctx { + int cur_op_index; // index of current op (-1 == no current entry) + bool last_op; // last op + void *host_addr; // user VA of the host buffer (unused for PA to PA transfers) + dma_addr_t src_pa; // source dma address (PA to PA transfers only) + dma_addr_t dev_addr; // + u64 offset; // + u64 xfer_size; // size to transfer in the next step. pages pinned for that step + // for zerocopy requests, no pinning for PA to PA transfers or MC to MC transfers + u64 remaining; // + } current_op; + + wait_queue_head_t sync_wait_queue; // wait queue for synchronous requests. + struct ndma_h2t_sync_wait_pool { + atomic64_t map; + u64 pool[64]; + } sync_wait_pool; + + struct list_head compl_spill_q_head; +}; + +/** + * credit limits for servicing request queue + * submit limit is ~64MB (256 * 64 * 4096) aka (256 * max pinned pages per transfer * 4096) + * thread limit is 1/4 of that + */ +#define NDMA_ZC_SUBMIT_CREDIT_LIMIT 256 +#define NDMA_ZC_THREAD_CREDIT_LIMIT 64 + +// FORWARDS +// +static void ndma_zca_dma_ioq_initiate_cleanup(struct ndma_h2t_dma_ioq *dma_ioq); +static void ndma_zca_cleanup_queued_requests(struct ndma_h2t_dma_ioq *dma_ioq, bool discard_async); +static int ndma_zca_process_queued_requests(struct ndma_h2t_dma_ioq *dma_ioq, u32 credit_limit, bool remote); +static void ndma_zca_zerocopy_unpin_pages(struct ndma_h2t_zcdma_context *dma_ctx, u64 *nr_pinned_pages); + +/** + * sync completion support + * + * sync zerocopy requests flow through the same request queue to context queue flow + * as async requests. sync requests can't use the completion queue to wait on (they + * have no sequence number to correlate the completion with), so instead they + * wait on sync completion markers via a completion queue. There's a small number (64) + * of these markers allocated via a bitmap. + * + */ + +#define NDMA_ZC_SYNC_COMPL_MARKER (0xfeedbeef) +#define NDMA_ZC_SYNC_MAKE_COMPL_MARKER(r) ((u64)NDMA_ZC_SYNC_COMPL_MARKER | ((u64)(r) << 32)) +#define NDMA_ZC_SYNC_CHK_COMPL_MARKER(p) ((*((u64*)(p)) & 0xffffffff) == NDMA_ZC_SYNC_COMPL_MARKER) +#define NDMA_ZC_SYNC_COMPL_MARKER_RET(p) (((*((u64*)(p)) >> 32) & 0xffffffff)) + +/** + * ndma_zca_sync_compl_alloc() + * + * Allocate a sync completion marker - used for waking + * syncrhonous zero copy requests and return status. When a + * syncrhonous zero copy request is recieved, it needs to be queued + * like async zero copy requests, so we need to allocate a marker/cookie + * to wait on. Simple spin loop waiting for a free entry in a bitmap. + * + * Called with the ring lock held for allocation. Frees don't need locking. Just atomic + */ +static u64 *ndma_zca_sync_compl_alloc(struct ndma_h2t_dma_ioq *dma_ioq) +{ + int pos; + + while (atomic64_read(&dma_ioq->sync_wait_pool.map) == ~0ULL) { + mutex_unlock(&dma_ioq->ring->h2t_ring_lock); + usleep_range(10, 10); + mutex_lock(&dma_ioq->ring->h2t_ring_lock); + if (fatal_signal_pending(current)) { + return NULL; + } + } + + pos = ffz(atomic64_read(&dma_ioq->sync_wait_pool.map)); + atomic64_or(BIT_ULL(pos), &dma_ioq->sync_wait_pool.map); + dma_ioq->sync_wait_pool.pool[pos] = 0ULL; + return &dma_ioq->sync_wait_pool.pool[pos]; +} + +static void ndma_zca_sync_compl_free(struct ndma_h2t_dma_ioq *dma_ioq, u64 *compl_marker) +{ + u32 pos = (compl_marker - dma_ioq->sync_wait_pool.pool); + atomic64_andnot(BIT_ULL(pos), &dma_ioq->sync_wait_pool.map); +} + +/* ndma_zca_dma_ioq_idle() + * + * check if a dma ioq is idle (no requests, no contexts, and completion spill queue list empty) + */ +static bool ndma_zca_dma_ioq_idle(struct ndma_h2t_dma_ioq *dma_ioq) +{ + return (list_empty(&dma_ioq->queued_req_head) && ndma_ctx_queue_is_empty(&dma_ioq->ctx_q) && list_empty(&dma_ioq->compl_spill_q_head)); +} + +/** + * ndma_zca_dma_ioq_from_bit() + * + * Fetch a dma_ioq structure from the active dma_ioq bitmap + */ +static inline struct ndma_h2t_dma_ioq * ndma_zca_dma_ioq_from_bit(struct neuron_device *nd, int bit) +{ + u32 eng_id; + u32 qid; + struct ndma_eng *eng; + struct ndma_ring *ring; + + ndhal->ndhal_ndmar.ndmar_ctx_queue_from_bit(bit, &eng_id, &qid); + eng = &nd->ndma_engine[eng_id]; + ring = &eng->queues[qid].ring_info; + return ring->h2t_dma_ioq; +} + +/** + * ndma_zca_compl_queue_avail() + * + * returns number of free entries in the completion queue. The legacy + * implementation didn't wrap the head index on the runtime side (grr), + * so there's a wrap check/fix and warning. + */ +static u32 ndma_zca_compl_queue_avail(struct ndma_h2d_compl_queue *compl_queue) +{ + neuron_h2d_dma_compl_queue_t *compl_queue_shared = compl_queue->compl_queue_shared; + u32 head = smp_load_acquire(&compl_queue_shared->head); + u32 avail_cnt; + + if (head > compl_queue->capacity_mask) { + pr_warn_once("warning legacy runtime using non-wrapping head/tail async IO completion queue implementation\n"); + head = head & compl_queue->capacity_mask; + } + avail_cnt = compl_queue->capacity_mask - ((compl_queue->tail - head) & compl_queue->capacity_mask); + return avail_cnt; +} + +/** + * ndma_zca_compl_queue_add() + * + * Add an entry to the tail of the completion queue (shared w/ user space) then + * update our local copy of tail along with the tail in the completion queue. + */ +static void ndma_zca_compl_queue_add(struct ndma_h2d_compl_queue *compl_queue, + u64 sequence_num, s64 compl_ret, void *context) +{ + neuron_h2d_dma_compl_queue_t *compl_queue_shared = compl_queue->compl_queue_shared; + neuron_h2d_dma_compl_queue_entry_t *entry = &compl_queue_shared->entries[compl_queue->tail]; + entry->compl_ret = compl_ret; + entry->context = context; + entry->sequence_num = sequence_num; + + compl_queue->tail = (compl_queue->tail + 1) & compl_queue->capacity_mask; + smp_store_release(&compl_queue_shared->tail, compl_queue->tail); +} + +static bool ndma_zca_compl_spill_rec_q_full(struct ndma_h2t_compl_spill_q_rec *spill_rec) +{ + return (spill_rec->hdr.head == (spill_rec->hdr.tail +1) % NDMA_H2T_COMPL_SPILL_REC_Q_CNT); +} + +static bool ndma_zca_compl_spill_rec_q_empty(struct ndma_h2t_compl_spill_q_rec *spill_rec) +{ + return (spill_rec->hdr.head == spill_rec->hdr.tail); +} + +/** + * ndma_zca_compl_spill_add() + * + * There was no room in the completion queue so add the completion to + * a spill record. There's a list of Spill records, each with a tiny + * queue o entries, all kept in order that the completions are received. + */ +static int ndma_zca_compl_spill_add(struct ndma_h2t_dma_ioq *dma_ioq, u64 sequence_num, s64 compl_ret, void *context) +{ + struct ndma_h2t_compl_spill_q_rec *spill_rec; + neuron_h2d_dma_compl_queue_entry_t *ent; + + if (!list_empty(&dma_ioq->compl_spill_q_head)) { + spill_rec = list_last_entry(&dma_ioq->compl_spill_q_head, struct ndma_h2t_compl_spill_q_rec, hdr.queuelist); + if (!ndma_zca_compl_spill_rec_q_full(spill_rec)) { + goto add_entry; + } + } + // allocate a new spill record + // + spill_rec = kzalloc(sizeof(*spill_rec), GFP_KERNEL); + if (spill_rec == NULL) { + pr_err("nd%02d: async io completion spill record allocation failure\n", dma_ioq->eng->nd->device_index); + return -ENOMEM; + } + list_add_tail(&spill_rec->hdr.queuelist, &dma_ioq->compl_spill_q_head); + +add_entry: + ent = &spill_rec->queue[spill_rec->hdr.tail]; + ent->sequence_num = sequence_num; + ent->compl_ret = compl_ret; + ent->context = context; + spill_rec->hdr.tail = (spill_rec->hdr.tail + 1) % NDMA_H2T_COMPL_SPILL_REC_Q_CNT; + return 0; +} + +/** + * ndma_zca_compl_process_spill() + * + * If there wasn't room in the completion queue (user hasn't consumed async completions), + * we have to create spill records to hold additional completions to prevent the + * ctx queue (plus DMA ring) and request queue from all backing up. If spill is not empty, + * we try to drain it to the completion queue as space is avialable. + */ +static void ndma_zca_compl_process_spill(struct ndma_h2t_dma_ioq *dma_ioq) +{ + if (!list_empty(&dma_ioq->compl_spill_q_head)) { + struct ndma_h2d_compl_queue *compl_queue = &dma_ioq->ring->dma_compl_queue; + u32 compl_q_avail = ndma_zca_compl_queue_avail(compl_queue); + + while (!list_empty(&dma_ioq->compl_spill_q_head) && compl_q_avail) { + struct ndma_h2t_compl_spill_q_rec *spill_rec = list_first_entry(&dma_ioq->compl_spill_q_head, + struct ndma_h2t_compl_spill_q_rec, hdr.queuelist); + + while (compl_q_avail) { + neuron_h2d_dma_compl_queue_entry_t *ent; + if (ndma_zca_compl_spill_rec_q_empty(spill_rec)) { + list_del(&spill_rec->hdr.queuelist); + kfree(spill_rec); + break; + } + ent = &spill_rec->queue[spill_rec->hdr.head]; + ndma_zca_compl_queue_add(compl_queue, ent->sequence_num, ent->compl_ret, ent->context); + spill_rec->hdr.head = (spill_rec->hdr.head + 1) % NDMA_H2T_COMPL_SPILL_REC_Q_CNT; + compl_q_avail--; + } + } + } +} + +/** + * ndma_zca_compl_cleanup_spill() + * + * cleanup the spill record queue + * NOTE: only call from the completion thread + */ +static void ndma_zca_compl_cleanup_spill(struct ndma_h2t_dma_ioq *dma_ioq, bool discard_async) +{ + // first try to complete any requests + // + if (!discard_async) { + ndma_zca_compl_process_spill(dma_ioq); + } + + while (!list_empty(&dma_ioq->compl_spill_q_head)) { + struct ndma_h2t_compl_spill_q_rec *spill_rec = list_first_entry(&dma_ioq->compl_spill_q_head, + struct ndma_h2t_compl_spill_q_rec, hdr.queuelist); + list_del(&spill_rec->hdr.queuelist); + kfree(spill_rec); + } +} + +/** + * ndma_zca_release_ctx_resources() + * release resources allocated to the context and set the state to invalid. + * do not mess with the completion information or last flag, these are used after + * the resources are released. + */ +static void ndma_zca_release_ctx_resources(struct ndma_h2t_zcdma_context *ctx, u64 *nr_pinned_pages) +{ + // do not free or set completion_ptr null. it is managed by completion_pool in ctx queue + // do not free or set page_list null. it is managed by page_list_pool in ctx queue + + ndma_zca_zerocopy_unpin_pages(ctx, nr_pinned_pages); + ctx->nr_pages = 0; + ctx->prepin_proc = NULL; + ctx->state = NDMA_INVALID; +} + +/** + * ndma_zca_complete_request() + * + * for sync, wake the waiting process + * for async, put the completed request in the completion queue. If there's not space in the + * completion queue or the spill queue isn't empty, add the completion to the spill list queue + */ +static int ndma_zca_complete_request(struct ndma_h2t_dma_ioq *dma_ioq, u64 sequence_num, s64 compl_ret, void *context, bool discard_async) +{ + int ret = 0; + if (sequence_num == 0) { + *((u64*)context) = NDMA_ZC_SYNC_MAKE_COMPL_MARKER(compl_ret); + wake_up(&dma_ioq->sync_wait_queue); + } else if (!discard_async) { + struct ndma_h2d_compl_queue *compl_queue = &dma_ioq->ring->dma_compl_queue; + if (list_empty(&dma_ioq->compl_spill_q_head) && ndma_zca_compl_queue_avail(compl_queue)) { + ndma_zca_compl_queue_add(compl_queue,sequence_num, compl_ret, context); + } else { + ret = ndma_zca_compl_spill_add(dma_ioq, sequence_num, compl_ret, context); + } + } + return ret; +} + +/** + * ndma_zca_dma_ioq_cleanup_ctx_queue() + * + * cleanup the dma context queue (and descriptor ring). Once + * we hit a timeout, we assume the dma ring is wedged and we can just clear out the + * remaining ctx. Once dma_ioq->cleanup has been set, no more ctx queue entries will + * be created + */ +static void ndma_zca_dma_ioq_cleanup_ctx_queue(struct ndma_h2t_dma_ioq *dma_ioq) +{ + int ret = 0; + + while (!ndma_ctx_queue_is_empty(&dma_ioq->ctx_q)) { + struct ndma_h2t_zcdma_context * head_ctx = ndma_ctx_queue_peek_head(&dma_ioq->ctx_q); + + if (head_ctx->state == NDMA_COMPLETED) { + u64 dummy; + ndma_zca_release_ctx_resources(head_ctx, &dummy); + if (head_ctx->last) { + ndma_zca_complete_request(dma_ioq, head_ctx->sequence_num, head_ctx->compl_ret, head_ctx->context, false); + } + } else if (head_ctx->state == NDMA_SUBMITTED) { + if (ret != -ETIMEDOUT) { + // consume completions as long as we don't see a timeout. + ret = ndma_memcpy_wait_for_completion(dma_ioq->eng, dma_ioq->ring, head_ctx->nr_tx_desc + 1, + head_ctx->nr_rx_desc + 1, + head_ctx->completion_ptr, true, false); + } + head_ctx->compl_ret = ret; + ndma_zca_release_ctx_resources(head_ctx, &dma_ioq->ctx_q.nr_pinned_pages); + if (head_ctx->last) { + ndma_zca_complete_request(dma_ioq, head_ctx->sequence_num, head_ctx->compl_ret, head_ctx->context, false); + } + } else { + pr_err("nd%02d: h2d dma completion queue entry has invalid state %d\n", dma_ioq->eng->nd->device_index, head_ctx->state); + } + ndma_ctx_queue_pop_head(&dma_ioq->ctx_q); + } +} + +/** + * ndma_zca_dma_ioq_process_ctx_queue_no_wait() + * + * + * Note: This is currently called w/o locking. Assumption is only a single thread owns both the completion queue + * (tail) and the and the head of the context queue + */ +static int ndma_zca_dma_ioq_process_ctx_queue_no_wait(struct ndma_h2t_dma_ioq *dma_ioq) +{ + int ret = 0; + + while (!ndma_ctx_queue_is_empty(&dma_ioq->ctx_q)) { + struct ndma_h2t_zcdma_context *head_ctx = ndma_ctx_queue_peek_head(&dma_ioq->ctx_q); + + if (head_ctx->state == NDMA_COMPLETED) { + u64 dummy; + ndma_zca_release_ctx_resources(head_ctx, &dummy); + if (head_ctx->last) { + ret = ndma_zca_complete_request(dma_ioq, head_ctx->sequence_num, head_ctx->compl_ret, head_ctx->context, false); + } + ndma_ctx_queue_pop_head(&dma_ioq->ctx_q); + } else if (head_ctx->state == NDMA_SUBMITTED) { + volatile u32 *dst = (volatile u32 *)(head_ctx->completion_ptr + DMA_COMPLETION_MARKER_SIZE); + volatile u32 *src = (volatile u32 *)head_ctx->completion_ptr; + u32 dst_val = READ_ONCE(*dst); + + if (dst_val == DMA_COMPLETION_MARKER) { + WRITE_ONCE(*dst, 0); // this isn't strictly necessary but it will detect improper reuse issues + WRITE_ONCE(*src, DMA_COMPLETION_MARKER); // this isn't strictly necessary but it will detect improper reuse issues + dma_ioq->compl_timeout_deadline = 0ull; // reset completion timeout deadline + ndma_ack_completed_desc(dma_ioq->eng, dma_ioq->ring, head_ctx->nr_tx_desc + 1, head_ctx->nr_rx_desc + 1); + + ndma_zca_release_ctx_resources(head_ctx, &dma_ioq->ctx_q.nr_pinned_pages); + if (head_ctx->last) { + ret = ndma_zca_complete_request(dma_ioq, head_ctx->sequence_num, head_ctx->compl_ret, head_ctx->context, false); + } + ndma_ctx_queue_pop_head(&dma_ioq->ctx_q); + } else { + // if completion deadline isn't set, set it, otherwise check it + if (dma_ioq->compl_timeout_deadline == 0ull) { + u64 dummy, wait; + ndhal->ndhal_ndma.ndma_get_wait_for_completion_time(head_ctx->nr_tx_desc, true, &dummy, &wait); + dma_ioq->compl_timeout_deadline = get_jiffies_64() + usecs_to_jiffies(wait) + 1; + } else if (get_jiffies_64() > dma_ioq->compl_timeout_deadline) { + NDMA_DIAG_DUMP_ENGINE(dma_ioq->eng, dma_ioq->ring, head_ctx->nr_tx_desc + 1, "h2d_timeout"); + pr_err("nd%02d: h2d completion timeout on device addr: %llx size: %llx sequenc_num: %llx", + dma_ioq->eng->nd->device_index, head_ctx->dev_addr, head_ctx->size, head_ctx->sequence_num); + ret = -ETIMEDOUT; + + ndma_zca_release_ctx_resources(head_ctx, &dma_ioq->ctx_q.nr_pinned_pages); + if (head_ctx->last) { + ret = ndma_zca_complete_request(dma_ioq, head_ctx->sequence_num, ret, head_ctx->context, false); + } + ndma_ctx_queue_pop_head(&dma_ioq->ctx_q); + } + break; + } + } else { + pr_err("nd%02d: h2d dma completion queue entry has invalid state %d\n", dma_ioq->eng->nd->device_index, head_ctx->state); + ret = -ENXIO; // invalid entry + } + } + return ret; +} + +#define for_each_set_dma_ioq_bit(bit, val, start) \ + for (_tmp = (start) ? (((val) >> (start)) | ((val) << (64 - (start)))) \ + : (val); \ + _tmp && ((bit = (__ffs(_tmp) + (start)) & 63), 1); \ + _tmp &= _tmp - 1) + +/** + * ndma_zca_process_ring_cmpltns() + * + * thread owns ring head and completion queue. No locks (or separate lock) + * + * Note: the thread can't touch a dma_ioq unless the bit for the queue is + * set in the bitmap. + */ +static int ndma_zca_process_ring_cmpltns(struct neuron_device *nd) +{ + int ret = 0; + u64 bitmap = atomic64_read(&nd->dma_cmpltn_thread.nonempty_ctxq_bitmap); + int start = nd->dma_cmpltn_thread.rr_start_bit; + int bit; + u64 _tmp; + + // process ring completions no wait and add to completion queue (no locking) I think this is doable + // + for_each_set_dma_ioq_bit(bit, bitmap, start) { + struct ndma_h2t_dma_ioq *dma_ioq = ndma_zca_dma_ioq_from_bit(nd, bit); + ret = ndma_zca_dma_ioq_process_ctx_queue_no_wait(dma_ioq); + if (ret) { + dma_ioq->cleanup = true; // should set discard flag + } + } + + // process completion queue spill records + // + for_each_set_dma_ioq_bit(bit, bitmap, start) { + struct ndma_h2t_dma_ioq *dma_ioq = ndma_zca_dma_ioq_from_bit(nd, bit); + ndma_zca_compl_process_spill(dma_ioq); + } + + // process any queued requests and check for idle and for external cleanup request + // + for_each_set_dma_ioq_bit(bit, bitmap, start) { + struct ndma_h2t_dma_ioq *dma_ioq = ndma_zca_dma_ioq_from_bit(nd, bit); + + if (mutex_trylock(&dma_ioq->ring->h2t_ring_lock)) { + ret = ndma_zca_process_queued_requests(dma_ioq, NDMA_ZC_THREAD_CREDIT_LIMIT, true); + if (ret) { + dma_ioq->cleanup = true; + } + // Don't clear the bitmap if cleanup is set. A clear bitmap bit is our interlock with the code that + // frees the dma_ioq + if (ndma_zca_dma_ioq_idle(dma_ioq) && !dma_ioq->cleanup) { + atomic64_andnot(BIT_ULL(bit), &nd->dma_cmpltn_thread.nonempty_ctxq_bitmap); + } + mutex_unlock(&dma_ioq->ring->h2t_ring_lock); + + } + } + + // if any queues have cleanup requests, perform cleanup. Need to refresh + // the bitmap because it could be stale and as mentioned above, it's our interlock w/ code the frees the dma_ioq + bitmap = atomic64_read(&nd->dma_cmpltn_thread.nonempty_ctxq_bitmap); + + for_each_set_dma_ioq_bit(bit, bitmap, start) { + struct ndma_h2t_dma_ioq *dma_ioq = ndma_zca_dma_ioq_from_bit(nd, bit); + if (dma_ioq->cleanup) { + mutex_lock(&dma_ioq->ring->h2t_ring_lock); + ndma_zca_dma_ioq_cleanup_ctx_queue(dma_ioq); + ndma_zca_cleanup_queued_requests(dma_ioq, true); + ndma_zca_compl_cleanup_spill(dma_ioq, false); + if (!ndma_zca_dma_ioq_idle(dma_ioq)) { + pr_err("unexpected: cleanup was unsuccessful"); + } + mutex_unlock(&dma_ioq->ring->h2t_ring_lock); + atomic64_andnot(BIT_ULL(bit), &nd->dma_cmpltn_thread.nonempty_ctxq_bitmap); + } + } + + // rotate start to next core (estimated) + // + nd->dma_cmpltn_thread.rr_start_bit = (start + 8) & 63; + + cpu_relax(); + if (need_resched()) + cond_resched(); + return ret; +} + +/** + * ndma_zca_dma_ioq_init() + * + */ +int ndma_zca_dma_ioq_init(struct ndma_eng *eng, struct ndma_ring *ring, struct ndma_h2t_dma_ioq **dma_ioq) +{ + int ret = 0; + struct ndma_h2t_dma_ioq *p; + + if (*dma_ioq != NULL) { + pr_err("trying to initialize dma ioq structure when previous not freed"); + return -EEXIST; + } + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (p == NULL) { + return -ENOMEM; + } + p->eng = eng; + p->ring = ring; + p->qid = ring->qid; + INIT_LIST_HEAD(&p->queued_req_head); + p->pin_retries = 0ull; + p->compl_timeout_deadline = 0ull; + p->current_op.cur_op_index = -1; + init_waitqueue_head(&p->sync_wait_queue); + atomic64_set(&p->sync_wait_pool.map, 0); + INIT_LIST_HEAD(&p->compl_spill_q_head); + + ret = ndma_ctx_queue_init(&p->ctx_q); + if (ret) { + kfree(p); + return ret; + } + *dma_ioq = p; + return 0; +} + +/** + * ndma_zca_dma_ioq_free() + */ +void ndma_zca_dma_ioq_free(struct ndma_h2t_dma_ioq **dma_ioq) +{ + struct ndma_h2t_dma_ioq *p = *dma_ioq; + if (p == NULL) { + return; + } + ndma_zca_dma_ioq_initiate_cleanup(p); + ndma_ctx_queue_free(p->eng, p->ring, &p->ctx_q); + kfree(p); + *dma_ioq = NULL; +} + +/** + * Credit calculating functions for each of the three resources needed to submit a request + */ +static u32 ndma_zca_ctx_q_avail(struct ndma_ctx_queue *ctx_q) +{ + u32 avail_cnt = ctx_q->capacity_mask - ((ctx_q->tail - ctx_q->head) & ctx_q->capacity_mask); + return avail_cnt; +} + +static u32 ndma_zca_pinned_avail(struct ndma_h2t_dma_ioq *dma_ioq, u32 pin_size) +{ + return (NDMA_CTX_QUEUE_MAX_PINNED_PAGES - dma_ioq->ctx_q.nr_pinned_pages) / pin_size; +} + +/** + * ndma_zca_ring_avail + * + * return the number of transfer steps that fit in the available descriptor + * entries, less what is reserved for legacy sync. @nr_descs is the worst case + * data descriptor count for one step; the +1 covers its completion descriptor. + */ +static u32 ndma_zca_ring_avail(struct ndma_h2t_dma_ioq *dma_ioq, u32 nr_descs) +{ + u32 tx_avail, rx_avail, tmp; + + ndma_zc_descs_available(dma_ioq->eng, dma_ioq->qid, &tx_avail, &rx_avail); + /* Paired-op capacity is bounded by whichever ring has less room. */ + tmp = (tx_avail < rx_avail) ? tx_avail : rx_avail; + if (tmp < NDMA_SYNC_XFER_OUTSTANDING_MAX) { + return 0; + } + return (tmp - NDMA_SYNC_XFER_OUTSTANDING_MAX) / (nr_descs + 1); +} + +/** + * ndma_zca_ring_has_credit + * + * return true iff both tx and rx rings have room for the requested slot + * counts after honoring the legacy sync reservation. Uses the same + * threshold as ndma_zca_ring_avail but checks each side independently + * against its own need. + */ +static bool ndma_zca_ring_has_credit(struct ndma_h2t_dma_ioq *dma_ioq, u32 tx_need, u32 rx_need) +{ + u32 tx_avail, rx_avail; + + ndma_zc_descs_available(dma_ioq->eng, dma_ioq->qid, &tx_avail, &rx_avail); + return tx_avail >= tx_need + NDMA_SYNC_XFER_OUTSTANDING_MAX && + rx_avail >= rx_need + NDMA_SYNC_XFER_OUTSTANDING_MAX; +} + +/** + * ndma_zca_zerocopy_unpin_pages() + * + */ +static void ndma_zca_zerocopy_unpin_pages(struct ndma_h2t_zcdma_context *dma_ctx, u64 *nr_pinned_pages) +{ + if (dma_ctx->state >= NDMA_PINNED_UNSUBMITTED && dma_ctx->nr_pages > 0) { + /* Only unpin if we pinned it ourselves (not pre-pinned memory) */ + if (!dma_ctx->prepin_proc) { + if (dma_ctx->direction) { + unpin_user_pages(dma_ctx->page_list, dma_ctx->nr_pages); + } else { + unpin_user_pages_dirty_lock(dma_ctx->page_list, dma_ctx->nr_pages, true); + } + } else { + kref_put(&dma_ctx->prepin_proc->refcount, ndma_pinned_mem_process_release); + } + *nr_pinned_pages -= dma_ctx->nr_pages; + } +} + +/** + * ndma_zca_dma_ctx_init_common() + * Init for dma ctx common fields + */ +static void ndma_zca_dma_ctx_init_common(struct ndma_h2t_dma_ioq *dma_ioq, struct ndma_h2t_queued_request *request, struct ndma_h2t_zcdma_context *ctx) +{ + ctx->eng = dma_ioq->eng; + ctx->ring = dma_ioq->ring; + ctx->nr_tx_desc = 0; // Set by ndma_build_n_issue_zc_descs()/ndma_zca_build_n_issue_pa_descs(). + ctx->nr_rx_desc = 0; + ctx->host_addr = NULL; + ctx->src_pa = 0; + ctx->mm = NULL; + ctx->pid = request->pid; + ctx->sequence_num = request->sequence_num; + ctx->context = request->context; + ctx->compl_ret = 0; +} + +/** + * ndma_zca_dma_ioq_current_op_update() + * This function is used to update the dma_ioq's current op context. + * current_op contains the context of the op of the current request we are processing for this + * h2t queue (batch requests can have multiple ops). + * If done with this request, return true + */ +static bool ndma_zca_dma_ioq_current_op_update(struct ndma_h2t_queued_req_op_ctx *current_op, struct ndma_h2t_queued_request *request) +{ + nrt_tensor_batch_op_t *op; + current_op->cur_op_index++; + + // PA to PA copies are a single op with both sides already resolved to dma addresses, + // so there are no pages to pin and no host buffer to walk. + if (request->type == NDMA_H2T_REQ_PA_MEMCPY) { + if (current_op->cur_op_index == 1) { + current_op->cur_op_index = -1; // reset index we're done w/ prev request and need a new one + return true; + } + current_op->host_addr = NULL; + current_op->src_pa = request->pa_memcpy.src_base; + current_op->dev_addr = request->pa_memcpy.dst_base; + current_op->offset = 0; + current_op->remaining = request->pa_memcpy.size; + current_op->xfer_size = min(current_op->remaining, NDMA_ZC_PA_XFER_MAX); + current_op->last_op = true; + return false; + } + + if (current_op->cur_op_index == request->memcpy.num_ops) { + current_op->cur_op_index = -1; // reset index we're done w/ prev request and need a new one + return true; + } + + op = &request->memcpy.ops[current_op->cur_op_index]; + current_op->host_addr = op->buffer; + current_op->dev_addr = request->memcpy.dev_base + op->offset; + current_op->offset = (unsigned long)current_op->host_addr & (PAGE_SIZE - 1); + current_op->remaining = op->size; + current_op->xfer_size = ndma_calc_zc_pin_size(current_op->remaining + current_op->offset); + current_op->last_op = current_op->cur_op_index == request->memcpy.num_ops - 1; + return false; +} + +/** + * ndma_zca_dma_ioq_initiate_cleanup + * + * notify the completion thread to cleanup any dma_ioq outstanding requests + * then wait for the thread to finish cleanup. The cleanup flag also + * blocks any new requests/contexts from entering the sausage grinder + * which is why the cleanup flag is protected by the mutex. It also prevents + * any further setting of the bitmap for this queue which serves as an interlock + * with the thread. + */ +static void ndma_zca_dma_ioq_initiate_cleanup(struct ndma_h2t_dma_ioq *dma_ioq) +{ + int bit = ndhal->ndhal_ndmar.ndmar_ctx_queue_bit(dma_ioq->eng->eng_id, dma_ioq->qid); + mutex_lock(&dma_ioq->ring->h2t_ring_lock); + dma_ioq->cleanup = true; + mutex_unlock(&dma_ioq->ring->h2t_ring_lock); + + while (atomic64_read(&dma_ioq->eng->nd->dma_cmpltn_thread.nonempty_ctxq_bitmap) & BIT_ULL(bit)) { + msleep(10); + } +} + +/** + * ndma_zca_discard_queued_request() + * + * release a request and any per-type resources it holds. The request must not + * be on the queue (see ndma_zca_free_queued_request() for that case). + */ +static void ndma_zca_discard_queued_request(struct ndma_h2t_queued_request *request) +{ + switch (request->type) { + case NDMA_H2T_REQ_DMA_MEMCPY: + mmput(request->memcpy.mm); + break; + case NDMA_H2T_REQ_DESC_EXECUTE: + case NDMA_H2T_REQ_BAR4_WR: + case NDMA_H2T_REQ_PA_MEMCPY: + break; + } + kvfree(request); +} + +static void ndma_zca_free_queued_request(struct ndma_h2t_queued_request *request) +{ + list_del(&request->queuelist); + ndma_zca_discard_queued_request(request); +} +/** + * ndma_zca_cleanup_queued_requests() + * + * cleanup any outstanding requests + * NOTE: only call from the completion thread + */ +static void ndma_zca_cleanup_queued_requests(struct ndma_h2t_dma_ioq *dma_ioq, bool discard_async) +{ + struct ndma_h2t_queued_request *request; + + if (current != dma_ioq->eng->nd->dma_cmpltn_thread.thread) { + pr_err("ERROR: cleanup not called from completion thread"); + } + + while (!list_empty(&dma_ioq->queued_req_head)) { + request = list_first_entry(&dma_ioq->queued_req_head, struct ndma_h2t_queued_request, queuelist); + + ndma_zca_complete_request(dma_ioq, request->sequence_num, -EIO, request->context, discard_async); + ndma_zca_free_queued_request(request); + } +} + +/** + * ndma_zca_push_user_descs() + * + * Copy a user-supplied descriptor block into the next slots of one side + * (tx or rx) of the ring, rewriting the ring-id/phase-bit per descriptor. + * User-supplied ring-id bits are always overwritten because the target + * slot's phase-bit is a function of the ring's current position, not + * whatever the caller happened to put there. + */ +static int ndma_zca_push_user_descs(struct ndma_eng *eng, u32 qid, + enum udma_type type, + const union udma_desc *src_buf, + u32 nr_desc) +{ + struct udma_q *q; + u32 i; + int ret; + + ret = udma_q_handle_get(&eng->udma, qid, type, &q); + if (ret) { + return ret; + } + + for (i = 0; i < nr_desc; i++) { + union udma_desc *slot = udma_desc_get(q); + + if (slot == NULL) { + pr_err("nd%02d: q%d %s: no descriptor slot available at %u/%u\n", + eng->nd->device_index, qid, + (type == UDMA_TX) ? "tx" : "rx", i, nr_desc); + return -ENOSPC; + } + *slot = src_buf[i]; + udma_ring_id_set(q, slot); + } + return 0; +} + +/** + * ndma_zca_build_n_issue_pa_descs() + * + * Build and issue the descriptors for one step of a PA to PA transfer + * (NDMA_H2T_REQ_PA_MEMCPY). Counterpart of ndma_build_n_issue_zc_descs() for + * contiguous transfers where both sides are already dma addresses: nothing is + * pinned and the descriptors come from the same chunking helper the legacy + * sync/async path uses. + */ +static int ndma_zca_build_n_issue_pa_descs(struct ndma_h2t_zcdma_context *dma_ctx) +{ + u64 bytes_issued; + u32 nr_data_descs; + int ret; + + ret = ndma_pa_build_n_issue_descs(dma_ctx->eng, dma_ctx->ring, dma_ctx->src_pa, dma_ctx->dev_addr, + dma_ctx->size, true, true, 0ull, NDMA_ZC_PA_DESCS_PER_XFER, + dma_ctx->completion_ptr, &bytes_issued, &nr_data_descs, dma_ctx->last); + + /* paired m2s/s2m descriptors, so tx and rx counts match. */ + dma_ctx->nr_tx_desc = nr_data_descs; + dma_ctx->nr_rx_desc = nr_data_descs; + if (ret) { + return ret; + } + + // the step size is capped at NDMA_ZC_PA_XFER_MAX so the descriptor budget always covers it + if (bytes_issued != dma_ctx->size) { + pr_err("nd%02d: PA transfer issued %llu of %llu bytes\n", + dma_ctx->eng->nd->device_index, bytes_issued, dma_ctx->size); + return -EIO; + } + + dma_ctx->state = NDMA_SUBMITTED; + return 0; +} + +/** + * ndma_zca_process_desc_execute_request() + * + * Push a caller-built pair of tx/rx descriptor blocks onto the ring, + * append our own completion desc pair, and register a ctx queue entry + * so the shared completion thread will retire the transfer through the + * sync / async completion paths. + * + * Called with the ring lock held. Returns 0 on success (request has been + * consumed and freed), -EAGAIN when credits are insufficient (caller + * should retry later), or a negative errno on fatal error (request has + * been failed and freed). + */ +static int ndma_zca_process_desc_execute_request(struct ndma_h2t_dma_ioq *dma_ioq, + struct ndma_h2t_queued_request *request) +{ + struct ndma_h2t_zcdma_context *tail_ctx; + u32 tx_need = request->desc_execute.nr_tx_desc + 1; // +1 for completion desc + u32 rx_need = request->desc_execute.nr_rx_desc + 1; + int ret; + + // 1 ctx slot to route completion. + if (ndma_zca_ctx_q_avail(&dma_ioq->ctx_q) == 0) { + return -EAGAIN; + } + + if (!ndma_zca_ring_has_credit(dma_ioq, tx_need, rx_need)) { + return -EAGAIN; + } + + tail_ctx = ndma_ctx_queue_peek_tail(&dma_ioq->ctx_q); + ndma_zca_dma_ctx_init_common(dma_ioq, request, tail_ctx); + + // Push user tx descs then rx descs into the ring. + ret = ndma_zca_push_user_descs(dma_ioq->eng, dma_ioq->qid, UDMA_TX, + request->desc_execute.tx_desc_buf, + request->desc_execute.nr_tx_desc); + if (ret) { + goto fail; + } + ret = ndma_zca_push_user_descs(dma_ioq->eng, dma_ioq->qid, UDMA_RX, + request->desc_execute.rx_desc_buf, + request->desc_execute.nr_rx_desc); + if (ret) { + goto fail; + } + + // Append our completion desc pair on top of the user's descs. In the future, caller + // will pass the barrier request to the driver + ret = ndma_memcpy_add_completion_desc(dma_ioq->eng, dma_ioq->ring, tail_ctx->completion_ptr, UDMA_M2M_BARRIER_SOW); + if (ret) { + goto fail; + } + + // Kick the transfer (nr_*_desc + 1 accounts for our completion pair). + ret = udma_m2m_copy_start(&dma_ioq->eng->udma, dma_ioq->qid, tx_need, rx_need); + if (ret) { + goto fail; + } + + tail_ctx->direction = true; + tail_ctx->last = true; + tail_ctx->nr_pages = 0; + tail_ctx->nr_tx_desc = request->desc_execute.nr_tx_desc; + tail_ctx->nr_rx_desc = request->desc_execute.nr_rx_desc; + tail_ctx->state = NDMA_SUBMITTED; + ndma_ctx_queue_inc_tail(&dma_ioq->ctx_q); + ndma_zca_free_queued_request(request); + return 0; + +fail: + pr_err("nd%02d: desc_execute submission failed: %d\n", + dma_ioq->eng->nd->device_index, ret); + tail_ctx->state = NDMA_COMPLETED; + tail_ctx->direction = true; + tail_ctx->last = true; + tail_ctx->nr_pages = 0; + tail_ctx->compl_ret = ret; + ndma_ctx_queue_inc_tail(&dma_ioq->ctx_q); + ndma_zca_free_queued_request(request); + return ret; +} + +/** ndma_zca_retire_zero_len_op() + * + * Retire a zero length op without building a transfer for it. + * + * Such an op spends no credits and never advances, so leaving it to the submit loop + * would spin on it forever, and its nr_pages_max of 0 divides by zero in the credit + * math. It is retired through the context queue rather than completed inline so + * that ordering, the last op flag and completion behave as they do for a real + * transfer. + * + * Returns false if the context queue is full and the caller should retry later, + * true once the op is retired and the current op has been advanced. + */ +static bool ndma_zca_retire_zero_len_op(struct ndma_h2t_dma_ioq *dma_ioq, + struct ndma_h2t_queued_request *request, + struct ndma_h2t_queued_req_op_ctx *current_op) +{ + struct ndma_h2t_zcdma_context *tail_ctx; + + if (ndma_zca_ctx_q_avail(&dma_ioq->ctx_q) == 0) { + return false; + } + + tail_ctx = ndma_ctx_queue_peek_tail(&dma_ioq->ctx_q); + ndma_zca_dma_ctx_init_common(dma_ioq, request, tail_ctx); + tail_ctx->dev_addr = current_op->dev_addr; + tail_ctx->size = 0; + tail_ctx->direction = (request->type == NDMA_H2T_REQ_PA_MEMCPY) ? true : request->memcpy.direction; + tail_ctx->last = current_op->last_op; + tail_ctx->nr_pages = 0; + tail_ctx->state = NDMA_COMPLETED; + tail_ctx->compl_ret = 0; + ndma_ctx_queue_inc_tail(&dma_ioq->ctx_q); + + if (ndma_zca_dma_ioq_current_op_update(current_op, request)) { + ndma_zca_free_queued_request(request); + } + return true; +} + +/* ndma_zca_process_queued_requests() + * + * - entered with mutex held + * + */ +static int ndma_zca_process_queued_requests(struct ndma_h2t_dma_ioq *dma_ioq, u32 credit_limit, bool remote) +{ + int ret = 0; + struct ndma_h2t_queued_request *request = NULL; + struct ndma_h2t_queued_req_op_ctx *current_op = &dma_ioq->current_op; + u32 credits; + + // if cleanup has been initiated, don't generate any more contexts + if (dma_ioq->cleanup) { + return 0; + } + + // requests remain in the queue until their context queue entries are built and submitted + // + while (!list_empty(&dma_ioq->queued_req_head)) { + u32 ctx_credits; + u32 pin_credits; + u32 rng_credits; + u32 nr_pages_max; + u32 nr_descs_max; + bool pa_copy; + request = list_first_entry(&dma_ioq->queued_req_head, struct ndma_h2t_queued_request, queuelist); + + // check if we're processing a new request + if (current_op->cur_op_index == -1) { + // Bar4 writes simply get a completion queue entry marked completed + // + if (request->type == NDMA_H2T_REQ_BAR4_WR) { // TODO: In theory individual OPs could be bar4 + struct ndma_h2t_zcdma_context *tail_ctx; + + if (ndma_zca_ctx_q_avail(&dma_ioq->ctx_q) == 0) { + goto done; + } + tail_ctx = ndma_ctx_queue_peek_tail(&dma_ioq->ctx_q); + ndma_zca_dma_ctx_init_common(dma_ioq, request, tail_ctx); + tail_ctx->direction = true; + tail_ctx->last = true; + tail_ctx->nr_pages = 0; + tail_ctx->state = NDMA_COMPLETED; + tail_ctx->compl_ret = request->bar4_wr.compl_ret; + ndma_ctx_queue_inc_tail(&dma_ioq->ctx_q); + ndma_zca_free_queued_request(request); + continue; + } + if (request->type == NDMA_H2T_REQ_DESC_EXECUTE) { + ret = ndma_zca_process_desc_execute_request(dma_ioq, request); + if (ret == -EAGAIN) { + // transient back-pressure: request stays on the + // queue and the completion thread will retry it + // once credits free up. Do not propagate to the + // submitter (would trigger a spurious cleanup). + ret = 0; + goto done; + } + if (ret) { + goto done; // fatal error; ctx entry already marked failed + } + continue; + } + // update the current op context for the new request + ndma_zca_dma_ioq_current_op_update(current_op, request); + } + + // a zero length op has no transfer to build, so retire it separately + // + if (current_op->remaining == 0) { + if (!ndma_zca_retire_zero_len_op(dma_ioq, request, current_op)) { + goto done; + } + continue; + } + + // compute how many contexts we can create/process for this op (min of all available credits we need so we know when to stop) + // it's possible we use less pin and ring credits so we could recompute in the loop + // + pa_copy = (request->type == NDMA_H2T_REQ_PA_MEMCPY); + if (pa_copy) { + // PA transfers pin nothing and are contiguous, so the descriptor count per + // step is driven by the transfer size, not the page count. + nr_descs_max = DIV_ROUND_UP(current_op->xfer_size, MAX_DMA_DESC_SIZE); + pin_credits = U32_MAX; + } else { + nr_pages_max = DIV_ROUND_UP(current_op->xfer_size, PAGE_SIZE); + nr_descs_max = nr_pages_max; // worst case: no two pinned pages are contiguous + pin_credits = ndma_zca_pinned_avail(dma_ioq, nr_pages_max); + } + ctx_credits = ndma_zca_ctx_q_avail(&dma_ioq->ctx_q); + rng_credits = ndma_zca_ring_avail(dma_ioq, nr_descs_max); + credits = min(min(min(ctx_credits, pin_credits), rng_credits), credit_limit); + + if (credits == 0) { + pr_info_once("no credits ctx: %d pin: %d rng %d limit %d\n", ctx_credits, pin_credits, rng_credits, credit_limit); + break; + } + + while (current_op->remaining && credits) { + struct ndma_h2t_zcdma_context *tail_ctx; + int nr_pages = pa_copy ? 0 : DIV_ROUND_UP(current_op->xfer_size, PAGE_SIZE); + + tail_ctx = ndma_ctx_queue_peek_tail(&dma_ioq->ctx_q); + ndma_zca_dma_ctx_init_common(dma_ioq, request, tail_ctx); + tail_ctx->host_addr = current_op->host_addr; + tail_ctx->src_pa = current_op->src_pa; + tail_ctx->dev_addr = current_op->dev_addr; + // First chunk may be unaligned; later chunks are page-aligned with offset=0. + tail_ctx->size = current_op->xfer_size - current_op->offset; + tail_ctx->direction = pa_copy ? true : request->memcpy.direction; + tail_ctx->last = (tail_ctx->size == current_op->remaining) && current_op->last_op; + tail_ctx->nr_pages = nr_pages; + tail_ctx->state = NDMA_UNPINNED; // TODO - could remove + if (remote && !pa_copy) { + tail_ctx->mm = request->memcpy.mm; + } + + ret = pa_copy ? 0 : ndma_zerocopy_pin_pages(dma_ioq->eng->nd->device_index, dma_ioq->ring->h2t_nc_id, &dma_ioq->ctx_q, tail_ctx, remote); // TODO nc_id unused? + if (ret) { + // if memory resources are tight, drop out of the loop and retry later + if (ret == -ENOMEM) { + pr_warn_once("nd%02d: pinning retry encountered", dma_ioq->eng->nd->device_index); + dma_ioq->pin_retries++; // TODO need to have a way to report these + ret = 0; // non-fatal - we will retry + } else { + pr_err("failed to pin pages for zero copy dma on nd %d\n", dma_ioq->eng->nd->device_index); // TODO cache nd + tail_ctx->state = NDMA_COMPLETED; + tail_ctx->nr_pages = 0; // zero out page count since we never pinned + tail_ctx->compl_ret = ret; + ndma_ctx_queue_inc_tail(&dma_ioq->ctx_q); + ndma_zca_free_queued_request(request); + } + goto done; + } + + // process requests + current_op->remaining -= tail_ctx->size; + if (pa_copy) { + current_op->src_pa += tail_ctx->size; + } else { + current_op->host_addr += tail_ctx->size; + } + current_op->dev_addr += tail_ctx->size; + current_op->xfer_size = (current_op->remaining < current_op->xfer_size) ? current_op->remaining : current_op->xfer_size; + current_op->offset = 0; + + ret = pa_copy ? ndma_zca_build_n_issue_pa_descs(tail_ctx) : ndma_build_n_issue_zc_descs(tail_ctx); + if (ret) { + ndma_zca_zerocopy_unpin_pages(tail_ctx, &dma_ioq->ctx_q.nr_pinned_pages); + pr_err("failed to build and issue descs\n"); + tail_ctx->state = NDMA_COMPLETED; + tail_ctx->nr_pages = 0; // zero out page count since there are no pinned pages + tail_ctx->compl_ret = ret; + ndma_ctx_queue_inc_tail(&dma_ioq->ctx_q); + ndma_zca_free_queued_request(request); + goto done; + } + + tail_ctx->state = NDMA_SUBMITTED; + ndma_ctx_queue_inc_tail(&dma_ioq->ctx_q); + + credits--; // TODO we could recompute here + credit_limit--; + + // if we are done w/ current op for this request, update to the next op. If there's no more ops + // remove the current request and exit the loop. + // + if (current_op->remaining == 0) { + if (ndma_zca_dma_ioq_current_op_update(current_op, request)) { + ndma_zca_free_queued_request(request); + break; + } + } + } + } + +done: + // notify the thread that there's work to do. (right now even for sync (sequence_num == 0) + if (!remote) { + int bit = ndhal->ndhal_ndmar.ndmar_ctx_queue_bit(dma_ioq->eng->eng_id, dma_ioq->qid); + atomic64_or(BIT_ULL(bit), &dma_ioq->eng->nd->dma_cmpltn_thread.nonempty_ctxq_bitmap); // set the bit for this queue + wake_up(&dma_ioq->eng->nd->dma_cmpltn_thread.wait_queue); + } + + return ret; +} + +/** + * ndma_zca_request_submit() + * + * Common tail end of the request submission paths. Validates ring ownership, + * allocates a sync completion marker for sync submissions, hands the request + * over to the queue and drives queue processing. For sync submissions + * (request->sequence_num == 0) it blocks until the transfer retires and + * returns its status; async submissions return as soon as the request is + * queued and report completion through the shared completion queue. + * + * On entry the caller owns @request. On return ownership has transferred to + * the queue, or the request has been released on the failure paths. + */ +static int ndma_zca_request_submit(struct neuron_device *nd, u32 nc_id, int qid, + struct ndma_ring *ring, struct ndma_h2t_queued_request *request) +{ + struct ndma_h2t_dma_ioq *dma_ioq = ring->h2t_dma_ioq; + const u64 sequence_num = request->sequence_num; + u64 *sync_compl_marker = NULL; + int ret; + + // Add the request to the request queue head then kick off queue processing + // + mutex_lock(&ring->h2t_ring_lock); + + if (!ndmar_h2t_ring_is_owner(ring, nc_id)) { + pr_err("nd%02d: attempting to use qid %d that was not assigned to nc %d\n", nd->device_index, qid, nc_id); + ret = -ENOENT; + goto err_unlock; + } + + if (sequence_num == 0) { + sync_compl_marker = ndma_zca_sync_compl_alloc(dma_ioq); + if (sync_compl_marker == NULL) { + ret = -EINTR; + goto err_unlock; + } + request->context = sync_compl_marker; + } + + // if we are cleaning up, don't keep stuffing the sausage grinder do this after the sequence_num + // marker acquire because it does a lock/unlock spin + // + if (dma_ioq->cleanup) { + if (sync_compl_marker != NULL) { + ndma_zca_sync_compl_free(dma_ioq, sync_compl_marker); + } + ret = -EIO; + goto err_unlock; + } + + // once added to the list the processing code owns and is responsible for the request + // + list_add_tail(&request->queuelist, &dma_ioq->queued_req_head); // TODO COULD HAVE A FLAG FOR SYNC REQ TO INSERT AT FRONT+1 + + ret = ndma_zca_process_queued_requests(dma_ioq, NDMA_ZC_SUBMIT_CREDIT_LIMIT, false); + + mutex_unlock(&ring->h2t_ring_lock); + + if (ret) { + pr_err("nd%02d: error queuing request. %d", nd->device_index, ret); + ndma_zca_dma_ioq_initiate_cleanup(dma_ioq); + } + + if (sequence_num == 0) { + /* wait_event_killable only unblocks on fatal signals (SIGKILL or a + * thread-group already flagged for exit). Benign signals — SIGINT, + * SIGCHLD, SIGALRM, SIGWINCH, etc. — are left to be delivered on the + * next return to userspace and do NOT tear down the shared queue. + * On a real termination we still must drain: the completion thread + * would otherwise write into sync_compl_marker's pool slot after we + * free it, corrupting whichever sync waiter reallocates the slot. */ + ret = wait_event_killable(dma_ioq->sync_wait_queue, NDMA_ZC_SYNC_CHK_COMPL_MARKER(sync_compl_marker)); + if (ret) { + pr_err("nd%02d: sync io request aborted by fatal signal. Initiating cleanup", nd->device_index); + ndma_zca_dma_ioq_initiate_cleanup(dma_ioq); + } + ret = NDMA_ZC_SYNC_COMPL_MARKER_RET(sync_compl_marker); + ndma_zca_sync_compl_free(dma_ioq, sync_compl_marker); + } + + return ret; + +err_unlock: + mutex_unlock(&ring->h2t_ring_lock); + ndma_zca_discard_queued_request(request); + return ret; +} + +/** + * ndma_zca_zerocopy_submit() + * + */ +int ndma_zca_zerocopy_submit(struct neuron_device *nd, + u32 nc_id, + const nrt_tensor_batch_op_t *ops, + u32 num_ops, + dma_addr_t dev_base, + int qid, + bool direction, + bool bar4_wr, + u64 sequence_num, + void *context) +{ + int ret = 0; + const int eng_id = ndhal->ndhal_ndmar.ndmar_get_h2t_eng_id(nd, nc_id); + struct ndma_eng *eng = &nd->ndma_engine[eng_id]; + struct ndma_queue *queue = &eng->queues[qid]; + struct ndma_ring *ring = &queue->ring_info; + struct ndma_h2t_queued_request *request = NULL; + + ret = ndma_h2d_create_cmpltn_thread(nd); + if (ret) { + return ret; + } + + // TODO for batch, need ncdev to allocate because it needs to does op vetting + // allocate request and copy request info to it. + request = kvzalloc(sizeof(*request) + num_ops * sizeof(*ops), GFP_KERNEL); + if (request == NULL) { + return -ENOMEM; + } + + request->type = bar4_wr ? NDMA_H2T_REQ_BAR4_WR : NDMA_H2T_REQ_DMA_MEMCPY; + request->sequence_num = sequence_num; + request->context = (request->sequence_num == 0) ? NULL : context; + request->pid = task_tgid_nr(current); + + if (!bar4_wr) { + struct mm_struct *mm = current->mm; + + /* The completion kthread pins pages against this mm long after this + * ioctl returns, so pin the mm's lifetime here. Use mmget_not_zero: + * if the owning process is already exiting, fail the submit rather + * than queue work against a dying mm. */ + if (!mm || !mmget_not_zero(mm)) { + kvfree(request); + return -ESRCH; + } + memcpy(request->memcpy.ops, ops, num_ops * sizeof(*ops)); + request->memcpy.num_ops = num_ops; + request->memcpy.dev_base = dev_base; + request->memcpy.direction = direction; + request->memcpy.mm = mm; + } + + return ndma_zca_request_submit(nd, nc_id, qid, ring, request); +} + +/** + * ndma_zca_mc_copy_available() + * + */ +bool ndma_zca_mc_copy_available(struct neuron_device *nd, struct mem_chunk *src_mc, struct mem_chunk *dst_mc) +{ + const u32 nc_id = ndma_mc_pair_to_nc(src_mc, dst_mc); + const int eng_id = ndhal->ndhal_ndmar.ndmar_get_h2t_eng_id(nd, nc_id); + const int qid = ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(nc_id); + struct ndma_ring *ring = &nd->ndma_engine[eng_id].queues[qid].ring_info; + + // the pipeline has no equivalent of the legacy path's reset window DMA retry, so + // architectures that need the retry keep using the legacy path. + if (!ndma_zerocopy_supported()) { + return false; + } + + // requests are queued on the h2t ring of the nc that owns it. If this nc doesn't own + // the ring there's no queue to submit to, so the caller falls back to the legacy path. + return ndmar_h2t_ring_is_owner(ring, nc_id) && (ring->h2t_dma_ioq != NULL); +} + +/** + * ndma_zca_mc_copy_submit() + * + */ +int ndma_zca_mc_copy_submit(struct neuron_device *nd, struct mem_chunk *src_mc, struct mem_chunk *dst_mc, + u64 src_offset, u64 dst_offset, u64 size, int qid, u64 sequence_num, void *context) +{ + const u32 nc_id = ndma_mc_pair_to_nc(src_mc, dst_mc); + const int eng_id = ndhal->ndhal_ndmar.ndmar_get_h2t_eng_id(nd, nc_id); + struct ndma_ring *ring = &nd->ndma_engine[eng_id].queues[qid].ring_info; + struct ndma_h2t_queued_request *request = NULL; + int ret; + + // a zero sized transfer has no op to build a context from, so it would sit on the + // request queue forever. Nothing to do, so complete it here. + if (size == 0) { + return 0; + } + + ret = ndma_h2d_create_cmpltn_thread(nd); + if (ret) { + return ret; + } + + request = kvzalloc(sizeof(*request), GFP_KERNEL); + if (request == NULL) { + return -ENOMEM; + } + + request->type = NDMA_H2T_REQ_PA_MEMCPY; + request->sequence_num = sequence_num; + request->context = (sequence_num == 0) ? NULL : context; + request->pid = task_tgid_nr(current); + request->pa_memcpy.src_base = ndma_mc_to_pa(src_mc) + src_offset; + request->pa_memcpy.dst_base = ndma_mc_to_pa(dst_mc) + dst_offset; + request->pa_memcpy.size = size; + + return ndma_zca_request_submit(nd, nc_id, qid, ring, request); +} + +/** + * ndma_zca_desc_execute_submit() + * + * Queue a pair of pre-built H2D DMA descriptor blocks copied in from + * userspace and drive them through the shared queued-request pipeline. + * Behaves like ndma_zca_zerocopy_submit() but skips page pinning and + * descriptor construction: the descriptors are supplied by the caller. + * sequence_num == 0 selects the sync flow (blocks and returns status); + * non-zero selects the async flow (returns immediately, completion is + * posted to the shared CQ with sequence_num/context echoed back). + */ +int ndma_zca_desc_execute_submit(struct neuron_device *nd, + u32 nc_id, + int qid, + void __user *tx_descs_user, + u64 tx_descs_bytes, + void __user *rx_descs_user, + u64 rx_descs_bytes, + u64 sequence_num, + void *context) +{ + int ret = 0; + const int eng_id = ndhal->ndhal_ndmar.ndmar_get_h2t_eng_id(nd, nc_id); + struct ndma_eng *eng = &nd->ndma_engine[eng_id]; + struct ndma_queue *queue = &eng->queues[qid]; + struct ndma_ring *ring = &queue->ring_info; + struct ndma_h2t_queued_request *request = NULL; + u32 nr_tx, nr_rx; + const u32 max_side = DMA_H2T_DESC_COUNT - NDMA_SYNC_XFER_OUTSTANDING_MAX - 1; + + // Validate sizes: non-zero, aligned to descriptor size, and small enough + // that a single request + our completion pair fits after the sync reservation. + if (tx_descs_bytes == 0 || rx_descs_bytes == 0 || + !IS_ALIGNED(tx_descs_bytes, sizeof(union udma_desc)) || + !IS_ALIGNED(rx_descs_bytes, sizeof(union udma_desc))) { + return -EINVAL; + } + nr_tx = tx_descs_bytes / sizeof(union udma_desc); + nr_rx = rx_descs_bytes / sizeof(union udma_desc); + if (nr_tx > max_side || nr_rx > max_side) { + return -E2BIG; + } + + ret = ndma_h2d_create_cmpltn_thread(nd); + if (ret) { + return ret; + } + + request = kvzalloc(sizeof(*request) + (nr_tx + nr_rx) * sizeof(union udma_desc), GFP_KERNEL); + if (!request) { + return -ENOMEM; + } + request->type = NDMA_H2T_REQ_DESC_EXECUTE; + request->sequence_num = sequence_num; + request->context = (sequence_num == 0) ? NULL : context; + request->pid = task_tgid_nr(current); + request->desc_execute.tx_desc_buf = &request->desc_execute.descs[0]; + request->desc_execute.rx_desc_buf = &request->desc_execute.descs[nr_tx]; + request->desc_execute.nr_tx_desc = nr_tx; + request->desc_execute.nr_rx_desc = nr_rx; + + if (copy_from_user(request->desc_execute.tx_desc_buf, tx_descs_user, tx_descs_bytes) || + copy_from_user(request->desc_execute.rx_desc_buf, rx_descs_user, rx_descs_bytes)) { + ret = -EFAULT; + goto err; + } + + // Ownership of the request (and its desc buffers) transfers to the queue. + return ndma_zca_request_submit(nd, nc_id, qid, ring, request); + +err: + kvfree(request); + return ret; +} + diff --git a/neuron_dma.h b/neuron_dma.h index d724bc1..b9311cb 100644 --- a/neuron_dma.h +++ b/neuron_dma.h @@ -173,7 +173,7 @@ int ndma_memcpy_add_completion_desc( struct ndma_eng *eng, struct ndma_ring *rin * Wait for completion by start transfer of a DMA between two host memory locations and polling * on the host memory for the data to be written. */ -int ndma_memcpy_wait_for_completion(struct ndma_eng *eng, struct ndma_ring *ring, u32 count, void * ptr, bool async, bool is_d2d); +int ndma_memcpy_wait_for_completion(struct ndma_eng *eng, struct ndma_ring *ring, u32 tx_count, u32 rx_count, void * ptr, bool async, bool is_d2d); /** * ndma_mc_pair_to_nc - Resolve the neuron core id for two memory chunks. @@ -201,87 +201,6 @@ dma_addr_t ndma_mc_to_pa(struct mem_chunk *mc); */ bool ndma_zerocopy_supported(void); -/** - * ndma_zerocopy_submit() - Perform a pipelined zero-copy DMA transfer. - * @nd: Neuron device whose DMA engine is used. - * @nc_id: Neuron core identifier owning the queue. - * @ops: Array of host buffer descriptors. - * @num_ops: Number of descriptors in @ops. - * @dev_base: Base device physical address for the transfer. - * @qid: Queue identifier to submit descriptors on. - * @direction: true for host-to-device, false for device-to-host. - * @sequence_num: sequence number under async submission; 0 for sync. - * @context: opaque context pointer returned in the completion queue for async submissions. - * - * DMA data between a user space virtual address range and a contiguous location in device memory. - * In order to do this, we need to know the physical pages are associated with - * the user virtual address range and we need to make sure those physical pages stay - * associated with the user virtual address range while the DMA is happening. - * - * How do we do this? By asking the kernel to pin the physical pages in memory until we are - * done with them. But our transaction could be large, the physical pages won't be contiguous, - * and pinning takes CPU cycles, so we break the dma transfer up into a series of smaller transfers - * where we pipeline the pinning of physical pages with dma transfers. - * - * We use pin_user_pages_fast() to reduce pinning overhead because we know the process can't go - * away while we are down here doing our thing in the kernel within a single IOCTL call. - * - * ## For sync mode ## - * We enqueue DMA contexts into a fixed-size queue and drive submission from that queue. - * The loop: - * lock() - * while data remains - * submit any pinned-but-unsubmitted ctxs when descriptors are available - * create a new ctx when queue space and pin budget allow - * pin pages immediately for the ctx - * advance queue tail and update host/device pointers - * wait on submitted ctxs from the head as needed to relieve pressure - * submit remaining pinned ctxs - * drain submitted ctxs from the head - * unlock() - * - * ## For async mode ## - * We keep submitting dma contexts until we hit a threshold of pinned pages. - * Once we hit the threshold, we stop pinning pages and set the mm_struct for remote pinning later. - * TODO: liulily to add more detail once the complete async path is implemented. - * - * Notes: - * unpinning responsibilities. Up until a dma is successfully launched, this routine is responsible for unpinning - * host memory. After that ndma_zerocopy_wait_for_completion() owns responsibility for unpinning pages. - * - * We don't do this here, but pinning user pages across system (IOCTL) calls has a number of additional requirements. - * We would have to cleanup any pinned pages when the process goes away, so any pinned pages have to get tracked in - * process context. - * - */ -int ndma_zerocopy_submit(struct neuron_device *nd, - u32 nc_id, - const nrt_tensor_batch_op_t *ops, - u32 num_ops, - dma_addr_t dev_base, - int qid, - bool direction, - u64 sequence_num, - void *context); - -/** - * ndma_zerocopy_submit_completed() - Enqueue completed work for ordered async CQE. - * - * Used only for async BAR4 writes and temporary fake-async D2D copy, where the work - * has already completed before we enqueue a dummy context. - * - * @nd: Neuron device. - * @nc_id: NeuronCore id that owns the H2D queue. - * @qid: H2D queue id. - * @sequence_num: Async sequence number returned to userspace. - * @compl_ret: Completion result to report in the CQE. - * @context: Userspace completion context to copy into the CQE. - * - * Return: 0 if queued, negative errno otherwise. - */ -int ndma_zerocopy_submit_completed(struct neuron_device *nd, u32 nc_id, int qid, - u64 sequence_num, s64 compl_ret, void *context); - /** * Pre-pinned host memory support * @@ -323,4 +242,100 @@ int ndma_unpin_host_memory(u64 va); */ void ndma_pinned_mem_cleanup_process(pid_t pid); +/** + * ndma_zca_zerocopy_submit() - Perform a pipelined zero-copy DMA transfer. + * + * @nd: Neuron device whose DMA engine is used. + * @nc_id: Neuron core identifier owning the queue. + * @ops: Array of host buffer descriptors. + * @num_ops: Number of descriptors in @ops. + * @dev_base: Base device physical address for the transfer. + * @qid: Queue identifier to submit descriptors on. + * @direction: true for host-to-device, false for device-to-host. + * @sequence_num: sequence number under async submission; 0 for sync. + * @context: opaque context pointer returned in the completion queue for async submissions. + * + */ +int ndma_zca_zerocopy_submit(struct neuron_device *nd, + u32 nc_id, + const nrt_tensor_batch_op_t *ops, + u32 num_ops, + dma_addr_t dev_base, + int qid, + bool direction, + bool bar4_wr, + u64 sequence_num, + void *context); + +/** + * ndma_zca_mc_copy_available() - Check whether a mem chunk to mem chunk copy can + * be driven through the pipelined dma path. + * + * @nd: Neuron device whose DMA engine would be used. + * @src_mc: source memory chunk of the transfer. + * @dst_mc: destination memory chunk of the transfer. + * + * Return: true if ndma_zca_mc_copy_submit() can be used for this pair, false if + * the caller must fall back to the legacy ndma_memcpy_mc() path. + */ +bool ndma_zca_mc_copy_available(struct neuron_device *nd, struct mem_chunk *src_mc, struct mem_chunk *dst_mc); + +/** + * ndma_zca_mc_copy_submit() - Copy data from one memory chunk to another through + * the pipelined dma path. + * + * @nd: Neuron device whose DMA engine is used. + * @src_mc: source memory chunk from which data should be copied. + * @dst_mc: destination memory chunk to which data should be copied. + * @src_offset: offset in the source from where copy should start. + * @dst_offset: offset in the destination. + * @size: copy size. + * @qid: Queue identifier to submit descriptors on. + * @sequence_num: sequence number under async submission; 0 for sync. + * @context: opaque context pointer returned in the completion queue for async submissions. + * + * Return: 0 if the transfer succeeds (sync) or was queued (async), a negative + * error code otherwise. + */ +int ndma_zca_mc_copy_submit(struct neuron_device *nd, struct mem_chunk *src_mc, struct mem_chunk *dst_mc, + u64 src_offset, u64 dst_offset, u64 size, int qid, u64 sequence_num, void *context); + +/** + * ndma_zca_dma_ioq_init() - initialize zerocopy async pipelined dma + * + */ +int ndma_zca_dma_ioq_init(struct ndma_eng *eng, struct ndma_ring *ring, struct ndma_h2t_dma_ioq **dma_ioq); + +/** + * ndma_zca_dma_ioq_free() - free and cleanup zercopy async pipelined dma + * + */ +void ndma_zca_dma_ioq_free(struct ndma_h2t_dma_ioq **dma_ioq); + +/** + * ndma_zca_desc_execute_submit() - Queue and execute a caller-built pair of + * TX/RX DMA descriptors on an H2T queue. + * + * @nd: Neuron device. + * @nc_id: NeuronCore that owns the target queue. + * @qid: H2T queue id. + * @tx_descs_user: userspace pointer to tx (m2s) descriptors. + * @tx_descs_bytes: byte size of tx_descs_user (multiple of sizeof(union udma_desc)). + * @rx_descs_user: userspace pointer to rx (s2m) descriptors. + * @rx_descs_bytes: byte size of rx_descs_user. + * @sequence_num: 0 for sync flow (blocks and returns transfer status); + * non-zero for async flow (returns immediately, completion + * is posted to the shared CQ with sequence_num/context). + * @context: opaque userspace pointer echoed in the CQE (async only). + */ +int ndma_zca_desc_execute_submit(struct neuron_device *nd, + u32 nc_id, + int qid, + void __user *tx_descs_user, + u64 tx_descs_bytes, + void __user *rx_descs_user, + u64 rx_descs_bytes, + u64 sequence_num, + void *context); #endif + diff --git a/neuron_dma_diag.c b/neuron_dma_diag.c new file mode 100644 index 0000000..8aa3fd2 --- /dev/null +++ b/neuron_dma_diag.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + */ + +/* + * neuron_dma_diag.c — DMA engine diagnostic dump module. + * + * Provides a reusable diagnostic dump that can be called from any DMA + * error path (scrub timeout, h2d timeout, etc.). The dump performs only + * register reads and formatted prints; it does not sleep, allocate, or + * take locks, so it is safe to invoke from error/timeout context. + * + * Callers should gate invocation through NDMA_DIAG_DUMP_ENGINE() (see + * neuron_dma_diag.h) so the dump — and all of its register reads — sits + * behind a dynamic debug static-key branch that is off by default. + */ + +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ + +#include +#include +#include + +#include "udma/udma.h" +#include "neuron_device.h" +#include "neuron_dma.h" +#include "neuron_ring.h" +#include "neuron_reg_access.h" +#include "neuron_dma_diag.h" + +/* + * Index into udma.gen_int_regs->secondary_iofic_ctrl[], i.e. which of the two + * secondary IOFIC groups a cause bit belongs to. + */ +enum ndma_diag_sec_grp { + NDMA_DIAG_SEC_GRP_A = 0, /* M2S error conditions */ + NDMA_DIAG_SEC_GRP_B = 1, /* S2M error conditions */ +}; + +struct ndma_diag_error_bit { + u8 grp; /* enum ndma_diag_sec_grp */ + u8 bit; /* bit position within that group's int_cause_grp */ + const char *name; +}; + +/* + * Secondary IOFIC error cause bits, keyed on (group, bit position) exactly as + * the driver's own interrupt unmask code defines them -- see the + * INT_2ND_GROUP_A_M2S_* / INT_2ND_GROUP_B_S2M_* macros in udma/udma_iofic.c, + * which are the source of truth for this table. + * + * Do not translate these into the flat SDMA INTC trigger numbering used by the + * SoC bringup validation tables. That numbering is not a uniform offset per + * group: group A occupies only 29 flat slots (its bits 0-28 are triggers + * 58-86) while group B bit 0 is trigger 87, so a single base applied to both + * groups silently misdecodes every S2M cause. Keeping (group, bit) here means + * this table cannot drift out of step with udma_iofic.c. + * + * Only error conditions likely to indicate a DMA failure root cause are listed; + * gaps are bit positions the hardware does not define, or non-error hints. + */ +static const struct ndma_diag_error_bit known_error_bits[] = { + /* Group A -- M2S error conditions */ + { NDMA_DIAG_SEC_GRP_A, 0, "stream_completion_mismatch" }, + { NDMA_DIAG_SEC_GRP_A, 1, "stream_parity" }, + { NDMA_DIAG_SEC_GRP_A, 2, "stream_response" }, + { NDMA_DIAG_SEC_GRP_A, 3, "stream_timeout" }, + { NDMA_DIAG_SEC_GRP_A, 4, "completion_axi_parity" }, + { NDMA_DIAG_SEC_GRP_A, 5, "completion_axi_response" }, + { NDMA_DIAG_SEC_GRP_A, 6, "completion_axi_timeout" }, + { NDMA_DIAG_SEC_GRP_A, 7, "data_axi_parity" }, + { NDMA_DIAG_SEC_GRP_A, 8, "data_axi_response" }, + { NDMA_DIAG_SEC_GRP_A, 9, "data_axi_timeout" }, + { NDMA_DIAG_SEC_GRP_A, 10, "prefetch_axi_parity" }, + { NDMA_DIAG_SEC_GRP_A, 11, "prefetch_axi_response" }, + { NDMA_DIAG_SEC_GRP_A, 12, "prefetch_axi_timeout" }, + { NDMA_DIAG_SEC_GRP_A, 13, "packet_length" }, + { NDMA_DIAG_SEC_GRP_A, 14, "prefetch_max_descriptors" }, + { NDMA_DIAG_SEC_GRP_A, 17, "prefetch_ring_id_error" }, + { NDMA_DIAG_SEC_GRP_A, 26, "msix_timeout" }, + { NDMA_DIAG_SEC_GRP_A, 27, "msix_response" }, + /* Group B -- S2M error conditions */ + { NDMA_DIAG_SEC_GRP_B, 0, "data_axi_parity_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 1, "data_axi_response_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 2, "data_axi_timeout_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 6, "no_descriptors_timeout" }, + { NDMA_DIAG_SEC_GRP_B, 7, "no_descriptors_hint" }, + { NDMA_DIAG_SEC_GRP_B, 8, "prefetch_axi_parity_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 9, "prefetch_axi_response_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 10, "prefetch_axi_timeout_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 11, "prefetch_ring_id_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 13, "completion_axi_parity_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 14, "completion_axi_response_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 15, "completion_axi_timeout_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 16, "completion_stream" }, + { NDMA_DIAG_SEC_GRP_B, 17, "completion_unack" }, + { NDMA_DIAG_SEC_GRP_B, 18, "stream_header_parity" }, + { NDMA_DIAG_SEC_GRP_B, 19, "stream_header_error" }, + { NDMA_DIAG_SEC_GRP_B, 20, "stream_data_parity" }, + { NDMA_DIAG_SEC_GRP_B, 21, "stream_data" }, + { NDMA_DIAG_SEC_GRP_B, 24, "packet_length_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 26, "data_parity_s2m" }, + { NDMA_DIAG_SEC_GRP_B, 30, "prefetch_desc_parity_s2m" }, +}; + +/* --- Diagnostic dump sub-functions --- */ + +#define DIAG_PFX "NEURON_DMA_DIAG" + +static void ndma_diag_eng_state(struct ndma_eng *eng, + const char *reason) +{ + struct neuron_dma_eng_state state; + u32 m2s_state = 0, s2m_state = 0; + int ret; + + ret = ndmar_eng_get_state(eng->nd, eng->eng_id, &state); + if (ret) { + pr_err(DIAG_PFX "[%s] nd%02d %s: engine state read failed (%d)\n", + reason, eng->nd->device_index, eng->udma.name, ret); + return; + } + + /* Read raw M2S/S2M state registers for additional context */ + if (eng->udma.udma_regs_m2s && eng->udma.udma_regs_s2m) { + reg_read32(&eng->udma.udma_regs_m2s->m2s.state, &m2s_state); + reg_read32(&eng->udma.udma_regs_s2m->s2m.state, &s2m_state); + } + + pr_err(DIAG_PFX "[%s] nd%02d %s: tx_state=%u rx_state=%u rev_id=%u num_q=%u raw_m2s=0x%x raw_s2m=0x%x\n", + reason, eng->nd->device_index, eng->udma.name, + state.tx_state, state.rx_state, state.revision_id, + state.num_queues, m2s_state, s2m_state); +} + +static void ndma_diag_failing_queue(struct ndma_eng *eng, + struct ndma_ring *ring, + u32 desc_count, + const char *reason) +{ + struct neuron_dma_queue_state tx, rx; + struct udma_q *m2s_q, *s2m_q; + u32 tx_dcp, rx_dcp, tx_status_raw, rx_status_raw; + u32 tx_drhp_raw, rx_drhp_raw; + int ret; + + ret = ndmar_queue_get_state(eng->nd, eng->eng_id, ring->qid, &tx, &rx); + if (ret) { + pr_err(DIAG_PFX "[%s] nd%02d %s: q%d state read failed (%d)\n", + reason, eng->nd->device_index, eng->udma.name, ring->qid, ret); + return; + } + + pr_err(DIAG_PFX "[%s] nd%02d %s: q%d desc_count=%u TX hw_status=%u sw_status=%u head=%u tail=%u comp_head=%u length=%u base_addr=0x%llx\n", + reason, eng->nd->device_index, eng->udma.name, ring->qid, + desc_count, tx.hw_status, tx.sw_status, tx.head_pointer, + tx.tail_pointer, tx.completion_head, tx.length, + tx.base_addr); + pr_err(DIAG_PFX "[%s] nd%02d %s: q%d RX hw_status=%u sw_status=%u head=%u tail=%u comp_head=%u length=%u base_addr=0x%llx\n", + reason, eng->nd->device_index, eng->udma.name, ring->qid, + rx.hw_status, rx.sw_status, rx.head_pointer, + rx.tail_pointer, rx.completion_head, rx.length, + rx.base_addr); + + /* + * Read the Descriptor Current Pointer (DCP) — this tells us where + * the engine is currently *executing*, distinct from head (prefetched). + * If DCP == head, engine consumed all prefetched descs and is stalled + * waiting for prefetch. If DCP < head, data path itself is stalled. + * + * Also re-read the raw DRHP to capture the Ring ID bits [31:30]. + */ + m2s_q = &eng->udma.udma_q_m2s[ring->qid]; + s2m_q = &eng->udma.udma_q_s2m[ring->qid]; + + if (!m2s_q->q_regs || !s2m_q->q_regs) + return; + + tx_dcp = 0; rx_dcp = 0; + tx_drhp_raw = 0; rx_drhp_raw = 0; + tx_status_raw = 0; rx_status_raw = 0; + + reg_read32(&m2s_q->q_regs->rings.dcp, &tx_dcp); + reg_read32(&s2m_q->q_regs->rings.dcp, &rx_dcp); + reg_read32(&m2s_q->q_regs->rings.drhp, &tx_drhp_raw); + reg_read32(&s2m_q->q_regs->rings.drhp, &rx_drhp_raw); + reg_read32(&m2s_q->q_regs->rings.status, &tx_status_raw); + reg_read32(&s2m_q->q_regs->rings.status, &rx_status_raw); + + pr_err(DIAG_PFX "[%s] nd%02d %s: q%d TX dcp=%u drhp=0x%08x status=0x%08x RX dcp=%u drhp=0x%08x status=0x%08x\n", + reason, eng->nd->device_index, eng->udma.name, ring->qid, + tx_dcp & UDMA_M2S_Q_TDRHP_OFFSET_MASK, tx_drhp_raw, tx_status_raw, + rx_dcp & UDMA_M2S_Q_TDRHP_OFFSET_MASK, rx_drhp_raw, rx_status_raw); +} + +static void ndma_diag_interrupts(struct ndma_eng *eng, const char *reason) +{ + struct udma_iofic_regs __iomem *int_regs = eng->udma.gen_int_regs; + u32 grp[4]; + u32 sec[2]; + int ret, i; + + if (!int_regs) + return; + + /* Read all 4 main IOFIC groups */ + for (i = 0; i < 4; i++) { + ret = reg_read32(&int_regs->main_iofic.ctrl[i].int_cause_grp, &grp[i]); + if (ret) { + pr_err(DIAG_PFX "[%s] nd%02d %s: main_iofic group %d read failed (%d)\n", + reason, eng->nd->device_index, eng->udma.name, i, ret); + grp[i] = 0; + } + } + + pr_err(DIAG_PFX "[%s] nd%02d %s: main_iofic grp0=0x%08x grp1=0x%08x grp2=0x%08x grp3=0x%08x\n", + reason, eng->nd->device_index, eng->udma.name, + grp[0], grp[1], grp[2], grp[3]); + + /* Read secondary IOFIC (2 groups) */ + for (i = 0; i < 2; i++) { + ret = reg_read32(&int_regs->secondary_iofic_ctrl[i].int_cause_grp, &sec[i]); + if (ret) { + pr_err(DIAG_PFX "[%s] nd%02d %s: secondary_iofic group %d read failed (%d)\n", + reason, eng->nd->device_index, eng->udma.name, i, ret); + sec[i] = 0; + } + } + + if (!sec[0] && !sec[1]) + return; + + pr_err(DIAG_PFX "[%s] nd%02d %s: secondary_iofic grpA=0x%08x grpB=0x%08x\n", + reason, eng->nd->device_index, eng->udma.name, sec[0], sec[1]); + + /* + * Decode the named error causes. These live in the two secondary IOFIC + * groups, not in the main IOFIC -- the main IOFIC only carries a + * per-group summary bit for each of them. + */ + for (i = 0; i < ARRAY_SIZE(known_error_bits); i++) { + const struct ndma_diag_error_bit *eb = &known_error_bits[i]; + + /* Guard against a future table entry naming an unknown group */ + if (eb->grp >= ARRAY_SIZE(sec)) + continue; + + if (!(sec[eb->grp] & BIT(eb->bit))) + continue; + + pr_err(DIAG_PFX "[%s] nd%02d %s: grp%c bit %u: %s\n", + reason, eng->nd->device_index, eng->udma.name, + eb->grp == NDMA_DIAG_SEC_GRP_A ? 'A' : 'B', + eb->bit, eb->name); + } +} + +static void ndma_diag_prefetch_and_fifo(struct ndma_eng *eng, const char *reason) +{ + u32 pref_cfg1 = 0, pref_cfg2 = 0, pref_cfg3 = 0; + u32 data_cfg = 0, stream_cfg = 0, axi_ostand = 0; + + if (!eng->udma.udma_regs_m2s) + return; + + reg_read32(&eng->udma.udma_regs_m2s->m2s_rd.desc_pref_cfg_1, &pref_cfg1); + reg_read32(&eng->udma.udma_regs_m2s->m2s_rd.desc_pref_cfg_2, &pref_cfg2); + reg_read32(&eng->udma.udma_regs_m2s->m2s_rd.desc_pref_cfg_3, &pref_cfg3); + reg_read32(&eng->udma.udma_regs_m2s->m2s_rd.data_cfg, &data_cfg); + reg_read32(&eng->udma.udma_regs_m2s->axi_m2s.ostand_cfg, &axi_ostand); + reg_read32(&eng->udma.udma_regs_m2s->m2s.stream_cfg, &stream_cfg); + + pr_err(DIAG_PFX "[%s] nd%02d %s: pref_cfg=0x%x/0x%x/0x%x data_cfg=0x%x axi_ostand=0x%x stream_cfg=0x%x\n", + reason, eng->nd->device_index, eng->udma.name, + pref_cfg1, pref_cfg2, pref_cfg3, data_cfg, axi_ostand, stream_cfg); +} + +/** + * ndma_diag_dump_engine() - Dump DMA engine diagnostic state on error. + * + * Public entry point called from any DMA error/timeout path (normally via + * the dynamic-debug-gated NDMA_DIAG_DUMP_ENGINE() wrapper). Emits + * diagnostic state via pr_err. Must not sleep, allocate, or take locks. + * + * @eng: Pointer to the failing DMA engine (provides nd, eng_id, udma) + * @ring: Pointer to the failing ring (provides qid) + * @desc_count: Descriptor count that timed out + * @reason: Caller-provided tag for the dump, naming the code path + * (e.g. "dma_completion_timeout") + */ +void ndma_diag_dump_engine(struct ndma_eng *eng, struct ndma_ring *ring, + u32 desc_count, const char *reason) +{ + if (!eng || !ring || !eng->nd) { + pr_err(DIAG_PFX "[%s] invalid eng/ring/nd pointer, aborting dump\n", + reason ? reason : "unknown"); + return; + } + + ndma_diag_eng_state(eng, reason); + ndma_diag_failing_queue(eng, ring, desc_count, reason); + ndma_diag_interrupts(eng, reason); + ndma_diag_prefetch_and_fifo(eng, reason); +} + diff --git a/neuron_dma_diag.h b/neuron_dma_diag.h new file mode 100644 index 0000000..d585bf5 --- /dev/null +++ b/neuron_dma_diag.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + */ + +#ifndef NEURON_DMA_DIAG_H +#define NEURON_DMA_DIAG_H + +#include +#include + +struct ndma_eng; +struct ndma_ring; + +/** + * ndma_diag_dump_engine() - Dump DMA engine diagnostic state on error. + * + * Emits a concise diagnostic snapshot of the DMA engine via pr_err, + * covering engine state, failing queue, IOFIC interrupts, and + * prefetch/AXI configuration. Performs register reads and formatted + * prints only; it does not sleep, allocate, or take locks, so it is + * safe to call from error/timeout context. + * + * Prefer NDMA_DIAG_DUMP_ENGINE() (below) over calling this directly. + * + * @eng: Pointer to the failing DMA engine + * @ring: Pointer to the failing ring (provides qid) + * @desc_count: Descriptor count that timed out / failed + * @reason: Caller-provided string identifying the error context + * (e.g. "dma_completion_timeout", "h2d_timeout"). Name the + * code path, not one of its callers -- a shared path such as + * ndma_memcpy_wait_for_completion() is reached from scrub, + * memcpy and memset, so a caller-specific tag would mislabel + * most of its dumps. + */ +void ndma_diag_dump_engine(struct ndma_eng *eng, struct ndma_ring *ring, + u32 desc_count, const char *reason); + +/** + * NDMA_DIAG_DUMP_ENGINE() - dynamic-debug-gated DMA diagnostic dump. + * + * Wraps ndma_diag_dump_engine() in a dynamic debug callsite. When the + * callsite is disabled (the default) the static-key branch is not taken, + * the dump function is never entered, and none of its register reads + * execute -- so the fast path is unaffected. Enable at runtime, or before + * the driver loads, via the kernel dynamic debug facility, e.g.: + * + * # enable both diag callsites by their shared format prefix + * echo 'format "NEURON_DMA_DIAG dump" +p' \ + * > /sys/kernel/debug/dynamic_debug/control + * + * # or enable just one failure path by function + * echo 'func ndma_memcpy_wait_for_completion +p' \ + * > /sys/kernel/debug/dynamic_debug/control + * + * # pre-arm before the driver loads (modprobe.d or kernel cmdline) + * options neuron dyndbg="format 'NEURON_DMA_DIAG dump' +p" + * neuron.dyndbg="format 'NEURON_DMA_DIAG dump' +p" + * + * @reason must be a string literal -- it is folded into the callsite's + * dynamic debug format string. + */ +#define NDMA_DIAG_DUMP_ENGINE(eng, ring, desc_count, reason) \ +do { \ + DEFINE_DYNAMIC_DEBUG_METADATA(_ddm, "NEURON_DMA_DIAG dump: " reason); \ + if (DYNAMIC_DEBUG_BRANCH(_ddm)) \ + ndma_diag_dump_engine((eng), (ring), (desc_count), (reason)); \ +} while (0) + +#endif /* NEURON_DMA_DIAG_H */ + diff --git a/neuron_dmabuf.c b/neuron_dmabuf.c index 57dc738..3d6eb0c 100644 --- a/neuron_dmabuf.c +++ b/neuron_dmabuf.c @@ -311,3 +311,4 @@ int ndmabuf_get_fd(u64 va, u64 size, int *dmabuf_fd, u64 *offset) } #endif // #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) + diff --git a/neuron_dmabuf.h b/neuron_dmabuf.h index b369a3e..bf8064c 100644 --- a/neuron_dmabuf.h +++ b/neuron_dmabuf.h @@ -26,3 +26,4 @@ int ndmabuf_get_fd(u64 va, u64 size, int *fd, u64 *offset); #endif + diff --git a/neuron_ds.c b/neuron_ds.c index 85b989c..bea6960 100644 --- a/neuron_ds.c +++ b/neuron_ds.c @@ -275,3 +275,4 @@ uint64_t get_neuroncore_counter_value(struct neuron_datastore_entry *entry, int nc_index + NDS_MAX_NEURONCORE_COUNT); return 0; } + diff --git a/neuron_ds.h b/neuron_ds.h index 805e889..57576df 100644 --- a/neuron_ds.h +++ b/neuron_ds.h @@ -14,7 +14,7 @@ #include // Size of datastore -#define NEURON_DATASTORE_SIZE (256 * 1024UL) +#define NEURON_DATASTORE_SIZE (128 * 1024UL) #define NEURON_MAX_DATASTORE_ENTRIES_PER_DEVICE (NEURON_MAX_PROCESS_PER_DEVICE) struct neuron_device; @@ -117,3 +117,4 @@ void neuron_ds_release_lock(struct neuron_datastore *nds); uint64_t get_neuroncore_counter_value(struct neuron_datastore_entry *entry, int nc_index, int counter_index); #endif + diff --git a/neuron_fw_io.c b/neuron_fw_io.c index 13edbf3..cb5fc9a 100644 --- a/neuron_fw_io.c +++ b/neuron_fw_io.c @@ -324,6 +324,9 @@ static const u32 fw_io_cmd_timeout_tbl[FW_IO_CMD_MAX] = { (1000 * 1000 * 90), // cmd 3 (FW_IO_CMD_SET_POWER_PROFILE) (1000 * 1000 * 10), // cmd 4 (FW_IO_CMD_GET_DATA) (1000 * 1000 * 60), // cmd 5 (FW_IO_CMD_SET_FEATURE) + 0, // cmd 6 (currently unimplemented in Driver) + (1000 * 1000 * 20), // cmd 7 (FW_IO_CMD_SET_PCIE_PROFILE) + (1000 * 1000 * 20), // cmd 8 (FW_IO_CMD_SET_MAX_VERSION) - the ack has no spec'd latency bound: measured ~5s at load and ~10s after a device reset }; static const u32 fw_io_cmd_retry_tbl[FW_IO_CMD_MAX] = { @@ -333,6 +336,9 @@ static const u32 fw_io_cmd_retry_tbl[FW_IO_CMD_MAX] = { 3, // cmd 3 (FW_IO_CMD_SET_POWER_PROFILE) 3, // cmd 4 (FW_IO_CMD_GET_DATA) 3, // cmd 5 (FW_IO_CMD_SET_FEATURE) + 0, // cmd 6 (currently unimplemented in Driver) + 3, // cmd 7 (FW_IO_CMD_SET_PCIE_PROFILE) + 1, // cmd 8 (FW_IO_CMD_SET_MAX_VERSION) }; static u32 crc32c(const u8 *hdr, const u8 *data, size_t len) @@ -433,7 +439,8 @@ int fw_io_execute_request(struct fw_io_ctx *ctx, u8 command_id, const u8 *req, u return ret; } -int fw_io_execute_request_new(struct fw_io_ctx *ctx, u8 command_id, const u8 *req, u32 req_size, u8 *resp, u32 resp_size) +int fw_io_execute_request_new(struct fw_io_ctx *ctx, u8 command_id, const u8 *req, u32 req_size, + u8 *resp, u32 resp_size, u32 *fw_error_code) { int ret; int i, j; @@ -441,6 +448,8 @@ int fw_io_execute_request_new(struct fw_io_ctx *ctx, u8 command_id, const u8 *re u32 api_version_num = 0; + *fw_error_code = FW_IO_SUCCESS; + ret = fw_io_api_version_read(ctx->bar0, &api_version_num); if ((ret != 0) || (api_version_num < ndhal->ndhal_fw_io.new_readless_read_min_api_version)) { @@ -508,6 +517,8 @@ int fw_io_execute_request_new(struct fw_io_ctx *ctx, u8 command_id, const u8 *re continue; } + *fw_error_code = resp_header.hdr.error_code; + if (resp_header.hdr.error_code == FW_IO_SUCCESS) { u32 data_size = resp_header.hdr.size - sizeof(resp_header); if (data_size > 0 && resp != NULL) { @@ -527,18 +538,51 @@ int fw_io_execute_request_new(struct fw_io_ctx *ctx, u8 command_id, const u8 *re break; } - ctx->fw_io_err_count++; - pr_err("seq: %u, cmd: %u failed %u\n", ctx->next_seq_num, command_id, resp_header.hdr.error_code); ret = -EIO; if (resp_header.hdr.error_code == FW_IO_UNKNOWN_COMMAND) { + // expected while probing firmware that predates a command + // (e.g. SET_MAX_VERSION) - informational, not an error + if (command_id == FW_IO_CMD_SET_MAX_VERSION) { + pr_info_once("cmd: %u not supported by firmware\n", command_id); + } else { + ctx->fw_io_err_count++; + pr_err_ratelimited("seq: %u, cmd: %u failed %u\n", ctx->next_seq_num, command_id, resp_header.hdr.error_code); + } break; } + ctx->fw_io_err_count++; + pr_err("seq: %u, cmd: %u failed %u\n", ctx->next_seq_num, command_id, resp_header.hdr.error_code); } mutex_unlock(&ctx->lock); return ret; } +int fw_io_set_max_version(struct fw_io_ctx *ctx, u32 max_misc_ram_version) +{ + u32 fw_err; + int ret; + + ret = fw_io_execute_request_new(ctx, FW_IO_CMD_SET_MAX_VERSION, + (const u8 *)&max_misc_ram_version, + sizeof(max_misc_ram_version), NULL, 0, &fw_err); + if (ret && fw_err == FW_IO_UNKNOWN_COMMAND) + return -ENOTSUPP; + return ret; +} + +/* + * CRC32C (Castagnoli / iSCSI polynomial, init and final xor 0xffffffff) - the + * same convention the fw_io command channel uses. The firmware HBM layout + * reader (neuron_fw_layout.c) assumes firmware computes header_crc32 the same + * way; the golden vectors both sides must match live in KaenaDriverTest + * (test_fw_layout.cc). + */ +u32 fw_io_calc_crc32(const u8 *data, u32 len) +{ + return crc32c(NULL, data, len); +} + /** Read the list of addresses given in the address list and returns it's values in the value list * * @param ctx[in] - FWIO context @@ -622,6 +666,14 @@ int fw_io_read_csr_array_direct(void **addrs, u32 *values, u32 num_csrs, bool op int fw_io_read_csr_array_readless(void **ptrs, u32 *values, u32 num_csrs) { int i, j; + + // Virtual platforms don't model the readless-read mechanism, so they have to + // use plain MMIO instead. Keeping that here rather than in each arch's + // fw_io_read_csr_array() means a caller cannot pick the readless path on a + // platform that has no way to service it. + if (narch_is_qemu() || narch_is_emu()) + return fw_io_read_csr_array_direct(ptrs, values, num_csrs, true); + for (i = 0; i < MAX_REGIONS; i++) { void *start, *end; if (fwio_read_regions[i].ctx == NULL) { @@ -722,10 +774,12 @@ int fw_io_post_metric(struct fw_io_ctx *ctx, u8 *data, u32 size) int fw_io_post_metric_new(struct fw_io_ctx *ctx, u8 *data, u32 size) { + u32 fw_err; + if (size > FW_IO_REG_METRIC_BUF_SZ) { return -E2BIG; } - return fw_io_execute_request_new(ctx, FW_IO_CMD_POST_TO_CW, data, size, NULL, 0); + return fw_io_execute_request_new(ctx, FW_IO_CMD_POST_TO_CW, data, size, NULL, 0, &fw_err); } int fw_io_read_counters(struct fw_io_ctx *ctx, uint64_t addr_in[], uint32_t val_out[], @@ -847,6 +901,7 @@ void fw_io_get_total_ecc_err_counts(void *bar0, uint32_t *unrepairable_ecc_count int fw_io_set_power_profile(struct fw_io_ctx *ctx, uint32_t profile) { union fw_io_req_perfprofile_data data = {0}; + u32 fw_err; data.rec.profile = (uint8_t)profile; data.rec.voltage_margin = 0; data.rec.frequency_index = 0; @@ -856,13 +911,14 @@ int fw_io_set_power_profile(struct fw_io_ctx *ctx, uint32_t profile) return -EINVAL; } - return fw_io_execute_request_new(ctx, FW_IO_CMD_SET_POWER_PROFILE, (u8 *)&data, sizeof(data), NULL, 0); + return fw_io_execute_request_new(ctx, FW_IO_CMD_SET_POWER_PROFILE, (u8 *)&data, sizeof(data), NULL, 0, &fw_err); } int fw_io_get_performance_profile(struct fw_io_ctx *ctx, uint32_t *profile) { struct fw_io_get_data_request req = {0}; struct fw_io_get_perfprofile_response resp = {0}; + u32 fw_err; int ret; if (!ctx || !profile) { return -EINVAL; @@ -870,7 +926,7 @@ int fw_io_get_performance_profile(struct fw_io_ctx *ctx, uint32_t *profile) req.type = FW_IO_GET_DATA_PERF_PROFILE; - ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8 *)&req, sizeof(req.type), (u8 *)&resp, sizeof(resp)); + ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8 *)&req, sizeof(req.type), (u8 *)&resp, sizeof(resp), &fw_err); if (ret == 0) { *profile = (uint32_t)resp.profile; } else { @@ -892,6 +948,7 @@ int fw_io_enable_throttling_notifications(struct fw_io_ctx *ctx, bool enable) * This will avoid wiping out other enabled features when disabling one feature */ u8 features = 0; + u32 fw_err; if (!ctx) { return -EINVAL; @@ -901,13 +958,14 @@ int fw_io_enable_throttling_notifications(struct fw_io_ctx *ctx, bool enable) features = FW_IO_FEATURE_THROTTLING_NOTIFICATIONS; } - return fw_io_execute_request_new(ctx, FW_IO_CMD_SET_FEATURE, &features, sizeof(features), NULL, 0); + return fw_io_execute_request_new(ctx, FW_IO_CMD_SET_FEATURE, &features, sizeof(features), NULL, 0, &fw_err); } int fw_io_get_available_profiles(struct fw_io_ctx *ctx, u16 feature, u8 *num_profiles, u8 bitmap[32]) { struct fw_io_get_data_request req = {0}; struct fw_io_get_available_profiles_response response; + u32 fw_err; int ret; if (!ctx) { return -EINVAL; @@ -916,7 +974,7 @@ int fw_io_get_available_profiles(struct fw_io_ctx *ctx, u16 feature, u8 *num_pro req.type = FW_IO_GET_DATA_AVAILABLE_PROFILES; req.available_profiles.operation = feature; - ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8*)&req, sizeof(req.type) + sizeof(req.available_profiles), (u8*)&response, sizeof(response)); + ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8*)&req, sizeof(req.type) + sizeof(req.available_profiles), (u8*)&response, sizeof(response), &fw_err); if (ret) { return ret; } @@ -930,6 +988,7 @@ int fw_io_get_bar_info(struct fw_io_ctx *ctx, u8 query_type, struct fw_io_bar_en { struct fw_io_get_data_request req = {0}; struct fw_io_bar_entry_response resp[NEURON_MAX_SWITCH_BARS] = {0}; + u32 fw_err; int ret = 0; int i = 0; @@ -940,7 +999,7 @@ int fw_io_get_bar_info(struct fw_io_ctx *ctx, u8 query_type, struct fw_io_bar_en req.type = FW_IO_GET_DATA_BAR_INFO; req.bar_info.query_type = query_type; - ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8 *)&req, sizeof(req.type) + sizeof(req.bar_info), (u8 *)resp, sizeof(resp)); + ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8 *)&req, sizeof(req.type) + sizeof(req.bar_info), (u8 *)resp, sizeof(resp), &fw_err); if (ret) { return ret; } @@ -956,3 +1015,36 @@ int fw_io_get_bar_info(struct fw_io_ctx *ctx, u8 query_type, struct fw_io_bar_en return 0; } + +int fw_io_set_pcie_profile(struct fw_io_ctx *ctx, uint8_t profile) +{ + u32 fw_err; + + if (!ctx) { + return -EINVAL; + } + + return fw_io_execute_request_new(ctx, FW_IO_CMD_SET_PCIE_PROFILE, &profile, sizeof(profile), NULL, 0, &fw_err); +} + +int fw_io_get_pcie_profile(struct fw_io_ctx *ctx, uint8_t *profile) +{ + struct fw_io_get_data_request req = {0}; + uint8_t pcie_profile; + u32 fw_err; + int ret; + if (!ctx || !profile) { + return -EINVAL; + } + + req.type = FW_IO_GET_DATA_PCIE_PROFILE; + + ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8 *)&req, sizeof(req.type), &pcie_profile, sizeof(pcie_profile), &fw_err); + if (ret == 0) { + *profile = pcie_profile; + } else { + pr_err("failed to get pcie profile, ret = %d\n", ret); + } + + return ret; +} diff --git a/neuron_fw_io.h b/neuron_fw_io.h index 6c5924e..84ab2ac 100644 --- a/neuron_fw_io.h +++ b/neuron_fw_io.h @@ -80,6 +80,7 @@ enum fw_io_get_data_type { FW_IO_GET_DATA_PERF_PROFILE = 0x01, FW_IO_GET_DATA_AVAILABLE_PROFILES = 0x02, FW_IO_GET_DATA_BAR_INFO = 0x10, + FW_IO_GET_DATA_PCIE_PROFILE = 0x11, }; struct fw_io_get_data_request { @@ -143,7 +144,10 @@ enum { FW_IO_CMD_POST_TO_CW = 2, // post given blob as metrics to CloudWatch FW_IO_CMD_SET_POWER_PROFILE = 3, // set power profile FW_IO_CMD_GET_DATA = 4, // get various FW data (see fw_io_get_data_request) - FW_IO_CMD_SET_FEATURE = 5 // set feature bitmap + FW_IO_CMD_SET_FEATURE = 5, // set feature bitmap + FW_IO_CMD_SET_PCIE_PROFILE = 7, + FW_IO_CMD_SET_MAX_VERSION = 8, // advertise the max misc-ram version the driver supports; selects the firmware HBM layout version + FW_IO_CMD_MAX = 9 }; enum { @@ -293,8 +297,6 @@ struct fw_io_ctx { #define UINT64_LOW(x) ((u32)(((u64)(x)) & 0xffffffffULL)) #define UINT64_HIGH(x) ((u32)((x) >> 32)) -#define FW_IO_CMD_MAX 6 - // Wait up to 30 seconds in worst case. // Hardware can in some cases take longer to come out of reset but for some reads // (like reading device ID before creating the device) we cannot wait too long @@ -587,9 +589,30 @@ int fw_io_execute_request(struct fw_io_ctx *ctx, u8 command_id, const u8 *req, u * @param req_size: Request size * @param resp: Response buffer * @param resp_size: Response buffer size + * @param fw_error_code: set to the FW_IO_* error code from the last response received + * (FW_IO_SUCCESS if none was received) * @return 0 on success, negative on failure */ -int fw_io_execute_request_new(struct fw_io_ctx *ctx, u8 command_id, const u8 *req, u32 req_size, u8 *resp, u32 resp_size); +int fw_io_execute_request_new(struct fw_io_ctx *ctx, u8 command_id, const u8 *req, u32 req_size, + u8 *resp, u32 resp_size, u32 *fw_error_code); + +/** + * fw_io_set_max_version() - Advertise the highest misc-ram version the driver supports + * @param ctx: FWIO context + * @param max_misc_ram_version: highest misc-ram version whose metrics this driver parses + * @return 0 if firmware accepted the version, -ENOTSUPP if the running firmware does not + * implement SET_MAX_VERSION (predates the HBM layout feature), other negative + * value on failure + */ +int fw_io_set_max_version(struct fw_io_ctx *ctx, u32 max_misc_ram_version); + +/** + * fw_io_calc_crc32() - Compute a CRC32 with the polynomial shared with firmware + * @param data: data to checksum + * @param len: data length in bytes + * @return the CRC32 value + */ +u32 fw_io_calc_crc32(const u8 *data, u32 len); /** * fw_io_set_power_profile() - Set power profile @@ -633,4 +656,21 @@ int fw_io_get_available_profiles(struct fw_io_ctx *ctx, u16 feature, u8 *num_pro */ int fw_io_get_bar_info(struct fw_io_ctx *ctx, u8 query_type, struct fw_io_bar_entry *entries, u8 *count); +/** + * fw_io_set_pcie_profile() - Set pcie profile + * @param ctx: FWIO context + * @param profile: pcie profile value to set + * @return 0 on success, negative on failure + */ +int fw_io_set_pcie_profile(struct fw_io_ctx *ctx, uint8_t profile); + +/** + * fw_io_get_pcie_profile() - Get current pcie profile + * @param ctx: FWIO context + * @param profile: Pointer to store the current profile value + * @return 0 on success, negative on failure + */ +int fw_io_get_pcie_profile(struct fw_io_ctx *ctx, uint8_t *profile); + #endif + diff --git a/neuron_fw_layout.c b/neuron_fw_layout.c new file mode 100644 index 0000000..def2e8c --- /dev/null +++ b/neuron_fw_layout.c @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + * + * Reader for the firmware general memory layout (see neuron_fw_layout.h for + * the contract); the platform-specific parts come from ndhal_fw_metrics + * (vN/neuron_fw_metrics.c). All state transitions happen on the metrics + * thread via fw_layout_poll(); other threads only set renegotiate_requested + * or copy cached payloads out through fw_layout_get_entry(). + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "neuron_fw_layout.h" +#include "neuron_device.h" +#include "neuron_dhal.h" +#include "neuron_fw_io.h" +#include "neuron_arch.h" +#include "neuron_mempool.h" + +static_assert(sizeof(struct fw_layout_header) == FW_LAYOUT_HEADER_SIZE, "fw layout header must be 64 bytes"); +static_assert(sizeof(struct fw_layout_tlv) == FW_LAYOUT_TLV_SIZE, "fw layout descriptor must be 16 bytes"); + +static int fw_layout_enable = 1; +module_param(fw_layout_enable, int, S_IRUSR | S_IRGRP); +MODULE_PARM_DESC(fw_layout_enable, "enable the firmware HBM layout reader"); + +/* + * Copy region bytes [offset, offset + size) into a normal kernel buffer. HBM + * is read in whole 8-byte words; the head and tail are clipped here so callers + * need no alignment of their own. + */ +static void fw_layout_read(struct neuron_fw_layout *fwl, void *dst, u32 offset, u32 size) +{ + // TODO: we technically may have races with the head tail guard where: + // 1. we read old head + // 2. firmware writes new head and new body + // 3. we read new body and old tail + // Need more atomic reads to get around this + u32 end = offset + size; + u32 pos = round_down(offset, sizeof(u64)); // widest plain access: every HBM read is a PCIe round trip + u8 *d = dst; + + for (; pos < end; pos += sizeof(u64)) { + u64 word = readq(fwl->region + pos); + u32 from = max(pos, offset); + u32 to = min(pos + (u32)sizeof(u64), end); + + memcpy(d + (from - offset), (u8 *)&word + (from - pos), to - from); + } +} + +/* Free the parsed directory and the payload cache. */ +static void fw_layout_drop_layout(struct neuron_device *nd) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + struct fw_layout_entry *entries; + u8 *cache, *staging; + + mutex_lock(&fwl->lock); + entries = fwl->entries; + cache = fwl->cache; + staging = fwl->staging; + fwl->entries = NULL; + fwl->num_entries = 0; + fwl->cache = NULL; + fwl->staging = NULL; + fwl->cache_offset = 0; + fwl->cache_size = 0; + mutex_unlock(&fwl->lock); + + kfree(entries); + kvfree(cache); + kvfree(staging); +} + +/* Start a negotiation, invalidating any parsed layout (it belongs to the + * previous incarnation of the region). + */ +static void fw_layout_enter_negotiate(struct neuron_device *nd) +{ + fw_layout_drop_layout(nd); + nd->fw_layout.state = FW_LAYOUT_STATE_NEGOTIATE; +} + +/* SET_MAX_VERSION carries the platform's misc-ram version; firmware picks the + * layout version from it, so the header's ver_major says what we actually got. + */ +static void fw_layout_negotiate(struct neuron_device *nd) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + u32 version = ndhal->ndhal_fw_metrics.max_misc_ram_version; + int ret; + + ret = fw_io_set_max_version(nd->fw_io_ctx, version); + if (ret == 0) { + fwl->state = FW_LAYOUT_STATE_WAIT_HEADER; + return; + } + if (ret == -ENOTSUPP) { + // the running firmware predates SET_MAX_VERSION: there is no + // layout to read + pr_info("nd%d: firmware does not support the HBM layout, falling back to legacy behavior\n", + nd->device_index); + } else { + // FAIL response or timeout. No retry policy yet: give up on this + // incarnation of the layout, the next device reset negotiates again + pr_warn("nd%d: HBM layout negotiation (misc-ram version 0x%x) failed (%d), not reading the layout\n", + nd->device_index, version, ret); + } + fwl->state = FW_LAYOUT_STATE_IDLE; +} + +/* + * Validate the header in the prescribed order: magic -> INITIALIZED -> crc32 + * -> ver_major. Returns 0 with *hdr filled on success, -EAGAIN if the region + * is not (yet) valid or looks torn, -EINVAL on a major version we must not + * parse. + */ +static int fw_layout_validate_header(struct neuron_device *nd, struct fw_layout_header *hdr) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + u32 crc; + + fw_layout_read(fwl, hdr, 0, sizeof(*hdr)); + + if (hdr->magic != FW_LAYOUT_MAGIC) + return -EAGAIN; + if (!(hdr->flags & FW_LAYOUT_HDR_INITIALIZED)) + return -EAGAIN; // firmware is mid-(re)write + crc = fw_io_calc_crc32((const u8 *)hdr, FW_LAYOUT_HDR_CRC_LEN); + if (crc != hdr->header_crc32) + return -EAGAIN; // torn/invalid header + if (hdr->ver_major != FW_LAYOUT_VER_MAJOR) { + pr_warn("nd%d: fw layout ver %u.%u unsupported (driver supports major %u), not parsing\n", + nd->device_index, hdr->ver_major, hdr->ver_minor, FW_LAYOUT_VER_MAJOR); + return -EINVAL; + } + return 0; +} + +/* + * Walk the directory once, cache every descriptor and size the payload cache. + * Entries with a geometry we cannot safely read (empty, or payload outside the + * region) are kept but marked unusable so instance enumeration still matches + * directory order. + */ +static int fw_layout_parse_directory(struct neuron_device *nd, const struct fw_layout_header *hdr) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + struct fw_layout_entry *entries; + u32 count = hdr->record_count; + // payloads are densely packed after the full directory - a payload offset + // inside the header or the directory (even past the entries we cap at) + // can only be garbage + u64 dir_end = FW_LAYOUT_DIR_OFFSET + (u64)hdr->record_count * FW_LAYOUT_TLV_SIZE; + u32 dir_size; + u8 *dir; + u8 *cache = NULL, *staging = NULL; + u32 lo = U32_MAX, hi = 0; + u32 i; + + if (dir_end > FW_LAYOUT_REGION_SIZE) { + // the directory alone overflows the region: no layout version looks + // like this (a newer minor with more entries is capped below) + pr_err("nd%d: fw layout header claims %u entries, more than the region holds - firmware bug, not parsing\n", + nd->device_index, hdr->record_count); + return -EINVAL; + } + if (count > FW_LAYOUT_MAX_ENTRIES) { + pr_warn_once("nd%d: fw layout has %u entries, only reading the first %u\n", + nd->device_index, count, FW_LAYOUT_MAX_ENTRIES); + count = FW_LAYOUT_MAX_ENTRIES; + } + + dir_size = count * FW_LAYOUT_TLV_SIZE; + dir = kmalloc(dir_size, GFP_KERNEL); + entries = kcalloc(count, sizeof(*entries), GFP_KERNEL); + if (!dir || !entries) { + kfree(dir); + kfree(entries); + return -ENOMEM; + } + fw_layout_read(fwl, dir, FW_LAYOUT_DIR_OFFSET, dir_size); + + for (i = 0; i < count; i++) { + struct fw_layout_tlv tlv; + struct fw_layout_entry *e = &entries[i]; + + memcpy(&tlv, dir + i * FW_LAYOUT_TLV_SIZE, sizeof(tlv)); + e->type_id = tlv.type_id; + e->length = tlv.length; + e->offset = tlv.offset; + e->usable = tlv.length >= FW_LAYOUT_MIN_PAYLOAD_SIZE && + tlv.offset >= dir_end && + (u64)tlv.offset + tlv.length <= FW_LAYOUT_REGION_SIZE; + if (!e->usable) { + pr_warn_once("nd%d: fw layout entry %u (type 0x%04x) has unusable geometry (offset 0x%x len %u), skipping\n", + nd->device_index, i, tlv.type_id, tlv.offset, tlv.length); + continue; + } + lo = min(lo, tlv.offset); + hi = max(hi, tlv.offset + tlv.length); + } + kfree(dir); + + // the cache mirrors the one span that covers every usable payload. + // payloads are densely packed, so the span is about the sum of their + // sizes - and it is what every poll reads from HBM + if (hi <= lo) { + lo = 0; // no usable payloads: nothing to cache + hi = 0; + } else { + cache = kvzalloc(hi - lo, GFP_KERNEL); + staging = kvzalloc(hi - lo, GFP_KERNEL); + if (!cache || !staging) { + kvfree(cache); + kvfree(staging); + kfree(entries); + return -ENOMEM; + } + } + + // negotiation already dropped the previous incarnation; never leak one + fw_layout_drop_layout(nd); + mutex_lock(&fwl->lock); + fwl->entries = entries; + fwl->num_entries = count; + fwl->cache = cache; + fwl->staging = staging; + fwl->cache_offset = lo; + fwl->cache_size = hi - lo; + mutex_unlock(&fwl->lock); + return 0; +} + +/* Refresh the payload cache with one bulk read of its span. The HBM read + * lands in the staging buffer so the lock is only held for the memcpy. + */ +static void fw_layout_refresh_cache(struct neuron_device *nd) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + + fw_layout_read(fwl, fwl->staging, fwl->cache_offset, fwl->cache_size); + mutex_lock(&fwl->lock); + memcpy(fwl->cache, fwl->staging, fwl->cache_size); + mutex_unlock(&fwl->lock); +} + +/* WAIT_HEADER state: once the header validates, parse the directory and go + * ACTIVE. This is the only time the header is read. + */ +static void fw_layout_wait_header(struct neuron_device *nd) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + struct fw_layout_header hdr __aligned(sizeof(u64)); + int ret; + + ret = fw_layout_validate_header(nd, &hdr); + if (ret == 0) + ret = fw_layout_parse_directory(nd, &hdr); + + switch (ret) { + case 0: + pr_info("nd%d: fw layout ver %u.%u active, %u entries, caching %u bytes\n", + nd->device_index, hdr.ver_major, hdr.ver_minor, fwl->num_entries, + fwl->cache_size); + // fill the cache before serving it, so ACTIVE means real data + fw_layout_refresh_cache(nd); + fwl->state = FW_LAYOUT_STATE_ACTIVE; + break; + case -EINVAL: + case -ENOMEM: + // bad data from firmware, or no memory: idle until the next device reset + fwl->state = FW_LAYOUT_STATE_IDLE; + break; + default: + // -EAGAIN: not initialized yet or torn - keep polling the header + break; + } +} + +void fw_layout_poll(struct neuron_device *nd) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + + if (fwl->state == FW_LAYOUT_STATE_DISABLED) + return; + + // a completed device reset (the initial one at load included) kicks a + // negotiation: the carveout was scrubbed and re-written, so a parsed + // layout belongs to the previous incarnation of the region + if (atomic_xchg(&fwl->renegotiate_requested, 0)) + fw_layout_enter_negotiate(nd); + + // firmware halts the layout writers for any reset (TPB included) and + // scrubs the carveout on a device reset: nothing to read until the reset + // queue drains. a completed device reset kicks a negotiation (above) + if (nd->nr.req_pending_head) + return; + + switch (fwl->state) { + case FW_LAYOUT_STATE_NEGOTIATE: + fw_layout_negotiate(nd); + break; + case FW_LAYOUT_STATE_WAIT_HEADER: + fw_layout_wait_header(nd); + break; + case FW_LAYOUT_STATE_ACTIVE: + fw_layout_refresh_cache(nd); + break; + default: + // IDLE: nothing to read until the next device reset + break; + } +} + +void fw_layout_request_renegotiate(struct neuron_device *nd) +{ + atomic_set(&nd->fw_layout.renegotiate_requested, 1); +} + +int fw_layout_get_entry(struct neuron_device *nd, u16 type_id, u32 instance, void *buf, u32 buf_size) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + int ret = -ENOENT; + u32 i; + + mutex_lock(&fwl->lock); + if (fwl->state != FW_LAYOUT_STATE_ACTIVE) { + ret = -ENODEV; + goto done; + } + for (i = 0; i < fwl->num_entries; i++) { + struct fw_layout_entry *e = &fwl->entries[i]; + + if (e->type_id != type_id) + continue; + if (instance--) + continue; + if (!e->usable) { + ret = -ENODATA; + goto done; + } + ret = min(buf_size, e->length); + memcpy(buf, fwl->cache + (e->offset - fwl->cache_offset), ret); + goto done; + } +done: + mutex_unlock(&fwl->lock); + return ret; +} + +/* + * Locate the layout region within the BAR4 mapping. The region base is 1MB + * into HBM0 - inside the 16MB firmware carveout already excluded from the + * allocation pool. + */ +static int fw_layout_locate_region(struct neuron_device *nd) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + u64 hbm0_base, bar_offset; + int ret; + + if (!nd->npdev.bar4 || !nd->npdev.bar4_pa || !ndhal->ndhal_mmap.mmap_get_bar4_offset) + return -ENODEV; + + hbm0_base = ndhal->ndhal_mpset.device_dram_effective_base_addr[0] - MEMPOOL_CARVEOUT_SIZE; + ret = ndhal->ndhal_mmap.mmap_get_bar4_offset(hbm0_base + FW_LAYOUT_REGION_HBM_OFFSET, + FW_LAYOUT_REGION_SIZE, &bar_offset); + if (ret) + return ret; + if (bar_offset + FW_LAYOUT_REGION_SIZE > nd->npdev.bar4_size) + return -ENODEV; + + fwl->region = nd->npdev.bar4 + bar_offset; + return 0; +} + +void fw_layout_init(struct neuron_device *nd) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + + // nd is kvzalloc'ed, so all fw_layout fields start out zeroed + // (state == FW_LAYOUT_STATE_DISABLED). Do not memset here: the reset + // thread is already running and the initial reset may already have + // completed and set renegotiate_requested - a request that must not be + // wiped. + mutex_init(&fwl->lock); + + if (!fw_layout_enable) + return; + // simulation/emulation has no firmware behind the BARs + if (narch_is_qemu() || narch_is_emu()) + return; + if (fw_layout_locate_region(nd)) { + pr_info("nd%d: no BAR4 window for the fw layout, reader disabled\n", nd->device_index); + return; + } + + // the post-reset config (v3 onwards - v2 never negotiates) kicks a + // negotiation once the initial device reset completes. SET_MAX_VERSION is + // sent unconditionally - an old firmware answers UNKNOWN_COMMAND and the + // reader falls back to legacy behavior + fwl->state = FW_LAYOUT_STATE_IDLE; +} + +void fw_layout_destroy(struct neuron_device *nd) +{ + struct neuron_fw_layout *fwl = &nd->fw_layout; + + fw_layout_drop_layout(nd); + fwl->state = FW_LAYOUT_STATE_DISABLED; + fwl->region = NULL; + mutex_destroy(&fwl->lock); +} + diff --git a/neuron_fw_layout.h b/neuron_fw_layout.h new file mode 100644 index 0000000..4454f35 --- /dev/null +++ b/neuron_fw_layout.h @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + * + * Reader for the firmware general memory layout. + * + * Firmware publishes a self-describing, versioned region in HBM (inside the + * 16MB firmware carveout at the base of HBM0) instead of fixed misc-ram + * registers. The region contains a fixed 64-byte header, a directory of + * fixed-size TLV descriptors and densely packed payloads located by the + * directory offsets. + * + * This module implements only the layout protocol - it does not interpret a + * single payload byte. The per-type knowledge lives in + * neuron_fw_metrics_common.{c,h} and vN/neuron_fw_metrics.c. + * + * The driver's contract (reading side): + * 1. Negotiate: after every completed device reset (the initial one at load + * included) send SET_MAX_VERSION with the platform's misc-ram version + * (vN/neuron_fw_metrics.h). UNKNOWN_COMMAND means the running firmware + * predates the feature - fall back to legacy behavior. + * 2. Read and validate the header once per negotiation: + * magic -> INITIALIZED -> crc32 -> ver_major. + * 3. Walk the directory once and cache the offsets. + * 4. Poll from the metrics thread: refresh the payload cache - a mirror of + * the span of the region that holds every payload - with one bulk read. + * The cache is raw bytes; whether a payload is internally consistent + * (e.g. carries matching generation guards) is a property of its type, + * checked when a metric is parsed out of the cache. + * + * Not handled yet: a layout rebuilt or withdrawn under a running driver + * (firmware live update / rollback) and firmware that stopped publishing + * (staleness). + */ + +#ifndef NEURON_FW_LAYOUT_H +#define NEURON_FW_LAYOUT_H + +#include +#include +#include + +struct neuron_device; + +/* Layout major this driver parses (header ver_major check). Not the negotiated + * value - that is the platform's misc-ram version, ndhal_fw_metrics. + */ +#define FW_LAYOUT_VER_MAJOR 1 + +/* Region geometry: the layout occupies [0x100000, 0x200000) of HBM0, inside + * the 16MB firmware carveout the driver already reserves at the base of every + * HBM (MEMPOOL_CARVEOUT_SIZE). + */ +#define FW_LAYOUT_REGION_HBM_OFFSET 0x100000ull +#define FW_LAYOUT_REGION_SIZE 0x100000ull + +#define FW_LAYOUT_MAGIC 0x5043 /* "PC" */ +#define FW_LAYOUT_HDR_INITIALIZED (1u << 0) /* header flags: frame fully written */ + +#define FW_LAYOUT_HEADER_SIZE 64 +#define FW_LAYOUT_TLV_SIZE 16 +#define FW_LAYOUT_DIR_OFFSET FW_LAYOUT_HEADER_SIZE /* directory starts right after the header */ +#define FW_LAYOUT_HDR_CRC_LEN 12 /* header crc32 covers bytes [0x00..0x0B] */ + +/* Sanity caps on what we accept from the directory */ +#define FW_LAYOUT_MAX_ENTRIES 256 +#define FW_LAYOUT_MIN_PAYLOAD_SIZE 1 + +/* Fixed 64-byte header at the base of the region. All fields little-endian. */ +struct fw_layout_header { + u16 magic; /* 0x00 FW_LAYOUT_MAGIC */ + u16 flags; /* 0x02 bit 0 = INITIALIZED */ + u16 ver_major; /* 0x04 breaking change - do not parse on mismatch */ + u16 ver_minor; /* 0x06 additive - higher minor is safe to read */ + u32 record_count; /* 0x08 number of directory descriptors */ + u32 header_crc32; /* 0x0C crc32 over bytes [0x00..0x0B] */ + u8 reserved[48]; /* 0x10 */ +} __packed; + +/* Fixed 16-byte directory descriptor, record i at FW_LAYOUT_DIR_OFFSET + i * 16. */ +struct fw_layout_tlv { + u16 type_id; /* +0x00 payload schema id; unknown ids are skipped */ + u16 flags; /* +0x02 no flags defined yet */ + u32 length; /* +0x04 payload length in bytes */ + u32 offset; /* +0x08 payload offset from region base */ + u32 reserved; /* +0x0C */ +} __packed; + +/* Driver-side view of one directory entry. The payload bytes themselves live + * in the cache, at offset - cache_offset. + */ +struct fw_layout_entry { + u16 type_id; + u32 length; /* payload length in bytes */ + u32 offset; /* payload offset from region base */ + bool usable; /* geometry validated; payload lies within the cache */ +}; + +enum fw_layout_state { + FW_LAYOUT_STATE_DISABLED = 0, /* this driver instance cannot run the reader: feature off / no BAR4 */ + FW_LAYOUT_STATE_IDLE, /* nothing to read until the next device reset: negotiation failed, bad header or no memory */ + FW_LAYOUT_STATE_NEGOTIATE, /* send SET_MAX_VERSION */ + FW_LAYOUT_STATE_WAIT_HEADER, /* negotiated; waiting for a valid INITIALIZED header */ + FW_LAYOUT_STATE_ACTIVE, /* directory parsed; refreshing the payload cache */ +}; + +struct neuron_fw_layout { + /* the state machine is owned by the metrics thread; lock protects the + * entry table and the cache for cross-thread readers + */ + struct mutex lock; + enum fw_layout_state state; + atomic_t renegotiate_requested; /* set by the reset thread after a completed device reset */ + + void __iomem *region; /* layout region base within the BAR4 mapping */ + + struct fw_layout_entry *entries; /* one per directory descriptor */ + u32 num_entries; + + /* payload cache: a mirror of region bytes [cache_offset, cache_offset + + * cache_size) - the span covering every usable payload - + * refreshed with one bulk read per poll + */ + u8 *cache; + u32 cache_offset; + u32 cache_size; + u8 *staging; /* HBM read lands here off-lock, then memcpy into cache */ +}; + +/** + * fw_layout_init() - initialize the firmware layout reader for a device. + * + * Never fails: on unsupported setups (feature off, no BAR4, qemu/emulator) the + * reader is left in FW_LAYOUT_STATE_DISABLED. + * + * @nd: neuron device + */ +void fw_layout_init(struct neuron_device *nd); + +/** + * fw_layout_destroy() - free all reader resources. + * + * Must be called after the metrics thread is stopped. + * + * @nd: neuron device + */ +void fw_layout_destroy(struct neuron_device *nd); + +/** + * fw_layout_poll() - drive the layout reader state machine. + * + * Called periodically from the metrics thread while the device is READY. + * Negotiates with firmware, parses the layout and refreshes the payload cache. + * + * @nd: neuron device + */ +void fw_layout_poll(struct neuron_device *nd); + +/** + * fw_layout_request_renegotiate() - request a negotiation on the next poll. + * + * Called from the post-reset configuration after a completed device-level + * reset, the initial reset at driver load included: the HBM carveout is + * scrubbed and re-written, so a cached layout is only valid for one + * incarnation of the region. Only called on v3 onwards; elsewhere the reader + * stays IDLE. Safe to call from any context. + * + * @nd: neuron device + */ +void fw_layout_request_renegotiate(struct neuron_device *nd); + +/** + * fw_layout_get_entry() - copy the cached payload of an entry. + * + * Instances of a duplicated type_id are enumerated in directory order; per the + * layout contract, callers must identify a specific instance by the index + * field carried inside the payload (e.g. tpb_index), not by this position. + * + * The copy is a raw snapshot of the cache, taken under the lock so it cannot + * be torn by a refresh. Whether it is internally consistent (the head/tail + * generation guards of a gen-guarded type) is for the type's parser to check, + * see neuron_fw_metrics_common.h. + * + * @nd: neuron device + * @type_id: payload type to look up + * @instance: Nth descriptor with this type_id (0 based) + * @buf: destination buffer + * @buf_size: destination buffer size; the copy is truncated to this + * + * Return: number of bytes copied on success, -ENODEV if the layout is not + * active, -ENOENT if no such entry, -ENODATA if the entry's geometry is + * unusable and it is not cached. + */ +int fw_layout_get_entry(struct neuron_device *nd, u16 type_id, u32 instance, void *buf, u32 buf_size); + +#endif + diff --git a/neuron_fw_metrics_common.c b/neuron_fw_metrics_common.c new file mode 100644 index 0000000..1e9d8f1 --- /dev/null +++ b/neuron_fw_metrics_common.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + * + * Shared helpers for the firmware HBM layout metric payloads + * (see neuron_fw_metrics_common.h). + */ + +#include + +#include "neuron_fw_metrics_common.h" + +// both guards come from one cache snapshot rather than a fenced tail-then-head +// HBM read, so a write landing inside the ~us bulk read can pass. Accepted: +// rare against a ~2Hz writer, and it costs one 50ms sample +bool fw_metrics_gen_guard_ok(const u8 *payload, u32 length) +{ + return length >= 2 && payload[0] == payload[length - 1]; +} + diff --git a/neuron_fw_metrics_common.h b/neuron_fw_metrics_common.h new file mode 100644 index 0000000..70b6d32 --- /dev/null +++ b/neuron_fw_metrics_common.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + * + * Shared helpers for the metric payloads carried by the firmware HBM layout. + * The protocol (neuron_fw_layout.c) never interprets payload bytes; the + * parsers live per platform in vN/neuron_fw_metrics.c and build on these. + */ + +#ifndef NEURON_FW_METRICS_COMMON_H +#define NEURON_FW_METRICS_COMMON_H + +#include + +/** + * fw_metrics_gen_guard_ok() - check the u8 generation guards of a payload + * @payload: payload bytes as copied out of the layout cache + * @length: payload length in bytes + * + * Gen-guarded types carry the same generation byte first and last. A mismatch + * means the copy raced a firmware write: the parse fails for this tick. + * + * Return: true if the guards match. + */ +bool fw_metrics_gen_guard_ok(const u8 *payload, u32 length); + +#endif + diff --git a/neuron_ham_throttle.c b/neuron_ham_throttle.c new file mode 100644 index 0000000..2faa44f --- /dev/null +++ b/neuron_ham_throttle.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + */ + +/** + * HAM Periodic Throttle Notification + * + * This module sets up a notification queue per NC to receive periodic + * HAM throttle notifications. FW enables periodic mode on the HAM; + * this module handles the Neuron-side queue setup and routing. + * + * The parsed per-NC state is exposed to userspace through the + * neuron_sysfs_metrics framework (neuron_core{N}/stats/ham_throttle/throttle). + */ + +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ + +#include +#include + +#include "neuron_ham_throttle.h" +#include "neuron_device.h" +#include "neuron_arch.h" +#include "neuron_dhal.h" +#include "neuron_nq.h" +#include "neuron_mempool.h" +#include "neuron_reg_access.h" +#include "v3/notific.h" + +#define HAM_PERIODIC_NQ_SLOT 9 + +// Buffer size: 2KB, 128 entries at 16 bytes each. Periodic notifications +// arrive roughly every ~1s, and userspace polls and aggregates the most +// recent handful of entries, so a small ring is plenty. +#define HAM_PERIODIC_NQ_BUF_SIZE (2 * 1024) + +#define TPB_NOTIFIC_SUBBLOCK_OFFSET 0x1000 +#define TPB_SW_QUEUE_NUM4_OFFSET 0xa10 +#define NOTIFIC_WR_BUF_ENABLE_LO_OFFSET 0x30 +#define NOTIFIC_NQ_SW_OVERFLOW_OFFSET 0x18 + +#define NOTIFIC_WR_BUF_HAM_BIT 12 + +#define SW_QUEUE_NUM4_HAM_PERIODIC_MASK 0xF +#define SW_QUEUE_NUM4_HAM_PERIODIC_EN_BIT (1 << 16) + +#define NOTIF_ENTRY_SIZE 16 +#define HAM_PERIODIC_NQ_NUM_ENTRIES (HAM_PERIODIC_NQ_BUF_SIZE / NOTIF_ENTRY_SIZE) + +struct ham_throttle_entry { + u64 ts; + u8 mean; + u8 max; + u8 min; +}; + +/** + * ham_throttle_decode_entry() - decode one 16B ring entry at index @idx. + * + * Returns true and fills @out if the slot holds a real notification, false if + * the slot is empty (never written). + * + * Entry layout (16 bytes): + * word0 (d1): [7:0]=min, [15:8]=max, [23:16]=mean (0=throttled, 255=unthrottled) + * word1 (d2): high nibble 0xF = periodic marker + * word2/word3: 64-bit timestamp + */ +static bool ham_throttle_decode_entry(const u32 *buf_words, u32 idx, + struct ham_throttle_entry *out) +{ + const u32 *words = &buf_words[idx * 4]; + u32 d1; + + // An untouched slot reads as all-zero. A live entry always carries a + // non-zero timestamp, so treat a zero timestamp as "empty". + if (words[0] == 0 && words[1] == 0 && words[2] == 0 && words[3] == 0) + return false; + + out->ts = ((u64)words[3] << 32) | words[2]; + if (out->ts == 0) + return false; + + d1 = words[0]; + out->mean = (d1 >> 16) & 0xFF; + out->max = (d1 >> 8) & 0xFF; + out->min = d1 & 0xFF; + return true; +} + +/** + * ham_throttle_select_next() - find the most-recent entry strictly older than + * the previously selected one. + * + * We rank entries by the composite key (timestamp, slot index) so that even if + * two entries carry identical timestamps we still walk them one at a time and + * never pick the same slot twice. @prev_ts / @prev_idx describe the last entry + * we returned; pass (U64_MAX, INT_MAX) to get the newest entry. + * + * Returns true and fills @out + @out_idx if a matching entry was found. + */ +static bool ham_throttle_select_next(const u32 *buf_words, u64 prev_ts, int prev_idx, + struct ham_throttle_entry *out, int *out_idx) +{ + struct ham_throttle_entry cand; + u64 best_ts = 0; + int best_idx = -1; + u32 i; + + for (i = 0; i < HAM_PERIODIC_NQ_NUM_ENTRIES; i++) { + if (!ham_throttle_decode_entry(buf_words, i, &cand)) + continue; + + if (cand.ts > prev_ts || (cand.ts == prev_ts && (int)i >= prev_idx)) + continue; + + if (best_idx < 0 || cand.ts > best_ts || + (cand.ts == best_ts && (int)i > best_idx)) { + best_ts = cand.ts; + best_idx = i; + *out = cand; + } + } + + if (best_idx < 0) + return false; + *out_idx = best_idx; + return true; +} + +ssize_t neuron_ham_throttle_format(struct neuron_device *nd, int nc_id, char *buf, size_t len) +{ + struct mem_chunk *mc; + const u32 *buf_words; + struct ham_throttle_entry e; + u32 n_agg, count = 0; + u64 prev_ts = U64_MAX; + int prev_idx = INT_MAX; + u64 first_ts = 0, last_ts = 0; + u32 mean_sum = 0; + u8 min_of_min = 255, max_of_max = 0, mean_of_mean; + + if (nc_id < 0 || nc_id >= MAX_NC_PER_DEVICE) + return scnprintf(buf, len, "invalid nc\n"); + + mc = nd->ham_throttle.periodic_mc[nc_id]; + if (!mc || !mc->va) + return scnprintf(buf, len, "not configured\n"); + + buf_words = (const u32 *)mc->va; + + n_agg = nd->ham_throttle.agg_count[nc_id]; + if (n_agg == 0) + n_agg = 1; + if (n_agg > HAM_PERIODIC_NQ_NUM_ENTRIES) + n_agg = HAM_PERIODIC_NQ_NUM_ENTRIES; + + while (count < n_agg && + ham_throttle_select_next(buf_words, prev_ts, prev_idx, &e, &prev_idx)) { + prev_ts = e.ts; + + if (count == 0) + last_ts = e.ts; // newest entry seen + first_ts = e.ts; // oldest so far; final value after loop + + mean_sum += e.mean; + if (e.min < min_of_min) + min_of_min = e.min; + if (e.max > max_of_max) + max_of_max = e.max; + count++; + } + + if (count == 0) + return scnprintf(buf, len, "no entries\n"); + + mean_of_mean = mean_sum / count; + + return scnprintf(buf, len, + "entries=%u mean=%u/255(%d%%) max=%u/255(%d%%) min=%u/255(%d%%) first_ts=%llu last_ts=%llu\n", + count, + mean_of_mean, (mean_of_mean * 100) / 255, + max_of_max, (max_of_max * 100) / 255, + min_of_min, (min_of_min * 100) / 255, + first_ts, last_ts); +} + +ssize_t neuron_ham_throttle_set_aggregate_count(struct neuron_device *nd, int nc_id, + const char *buf, size_t size) +{ + u32 val; + int ret; + + if (nc_id < 0 || nc_id >= MAX_NC_PER_DEVICE) + return -EINVAL; + + ret = kstrtou32(buf, 0, &val); + if (ret) + return ret; + + if (val == 0 || val > HAM_PERIODIC_NQ_NUM_ENTRIES) { + pr_err("nd%d nc%d: aggregate count %u out of range [1, %u]\n", + nd->device_index, nc_id, val, HAM_PERIODIC_NQ_NUM_ENTRIES); + return -EINVAL; + } + + nd->ham_throttle.agg_count[nc_id] = val; + return size; +} + +/** + * ham_throttle_program_nc() - Program the NOTIFIC block for one NC to route + * periodic HAM notifications into slot 9. + * + * @nd: neuron device + * @nc_id: neuron core / TPB index + * @return: 0 on success, negative errno if a register read fails + */ +static int ham_throttle_program_nc(struct neuron_device *nd, u8 nc_id) +{ + u64 notific_offset = notific_get_relative_offset_v3(nc_id); + void *notific_base = nd->npdev.bar0 + notific_offset; + void *tpb_base = notific_base - TPB_NOTIFIC_SUBBLOCK_OFFSET; + void *sq4_addr = tpb_base + TPB_SW_QUEUE_NUM4_OFFSET; + u32 nq_en, val, after, wob, ovf; + int ret; + + // Enable only the HAM write-buffer input (bit 12), read-modify-write, so we + // don't disturb other notification sources the runtime depends on + ret = reg_read32(notific_base + NOTIFIC_WR_BUF_ENABLE_LO_OFFSET, &wob); + if (ret) { + pr_err("nd%d nc%d: read wr_buf_enable_lo failed: %d\n", + nd->device_index, nc_id, ret); + return ret; + } + wob |= (1 << NOTIFIC_WR_BUF_HAM_BIT); + reg_write32(notific_base + NOTIFIC_WR_BUF_ENABLE_LO_OFFSET, wob); + + // Enable the NQ slot for HAM periodic notifications + ret = reg_read32(notific_base + NOTIFIC_NQ_ENABLE_OFFSET, &nq_en); + if (ret) { + pr_err("nd%d nc%d: read nq_enable failed: %d\n", + nd->device_index, nc_id, ret); + return ret; + } + nq_en |= (1 << HAM_PERIODIC_NQ_SLOT); + reg_write32(notific_base + NOTIFIC_NQ_ENABLE_OFFSET, nq_en); + + // We don't continuously consume periodic HAM notifications. Let the queue + // wrap around and overwrite old entries. We are simply interested in the most + // recent entries + ret = reg_read32(notific_base + NOTIFIC_NQ_SW_OVERFLOW_OFFSET, &ovf); + if (ret) { + pr_err("nd%d nc%d: read nq_sw_overflow failed: %d\n", + nd->device_index, nc_id, ret); + return ret; + } + ovf |= (1 << HAM_PERIODIC_NQ_SLOT); + reg_write32(notific_base + NOTIFIC_NQ_SW_OVERFLOW_OFFSET, ovf); + + // Route periodic notifications to slot 9 and enable + ret = reg_read32(sq4_addr, &val); + if (ret) { + pr_err("nd%d nc%d: read sw_queue_num4 failed: %d\n", + nd->device_index, nc_id, ret); + return ret; + } + val = (val & ~SW_QUEUE_NUM4_HAM_PERIODIC_MASK) | HAM_PERIODIC_NQ_SLOT; + val |= SW_QUEUE_NUM4_HAM_PERIODIC_EN_BIT; + reg_write32(sq4_addr, val); + mb(); + reg_read32(sq4_addr, &after); + + pr_debug("nd%d nc%d sw_queue_num4 wrote=0x%08x after=0x%08x\n", + nd->device_index, nc_id, val, after); + return 0; +} + +/** + * neuron_ham_throttle_reprogram_nc() - Re-apply NOTIFIC routing for one NC. + * + * A TPB reset clears sw_queue_num4 and the NOTIFIC enables back to 0, losing + * the routing set up at init. The slot-9 buffer survives, so we only re-program + * the registers no re-allocation. + * + * @nd: neuron device + * @nc_id: neuron core / TPB index + */ +void neuron_ham_throttle_reprogram_nc(struct neuron_device *nd, u8 nc_id) +{ + if (narch_get_arch() != NEURON_ARCH_V3 && + narch_get_arch() != NEURON_ARCH_V4) + return; + if (nc_id >= MAX_NC_PER_DEVICE) + return; + // Only reprogram NCs whose buffer was already allocated at init + if (!nd->ham_throttle.periodic_mc[nc_id]) + return; + if (nd->ham_throttle.periodic_mc[nc_id]->va) + memset(nd->ham_throttle.periodic_mc[nc_id]->va, 0, HAM_PERIODIC_NQ_BUF_SIZE); + // The buffer survives reset, but nnq_set_hwaddr programs the NOTIFIC + // base-address/size registers, which the reset also clears + ndhal->ndhal_nq.nnq_set_hwaddr(nd, nc_id, HAM_PERIODIC_NQ_SLOT, + NQ_TYPE_THROTTLE, HAM_PERIODIC_NQ_BUF_SIZE, + nd->ham_throttle.periodic_mc[nc_id]->pa); + if (ham_throttle_program_nc(nd, nc_id)) + pr_warn("nd%d nc%d: failed to set periodic throttle routing\n", + nd->device_index, nc_id); +} + +int neuron_ham_throttle_init(struct neuron_device *nd) +{ + int ret; + u8 nc_id; + struct mem_chunk *mc = NULL; + + if (narch_get_arch() != NEURON_ARCH_V3 && + narch_get_arch() != NEURON_ARCH_V4) + return 0; + + for (nc_id = 0; nc_id < ndhal->ndhal_address_map.nc_per_device; nc_id++) { + if (((1 << nc_id) & ndhal->ndhal_address_map.dev_nc_map) == 0) + continue; + + // Reuse the existing buffer; only allocate the first time. This + // runs on each device-wide reset, so the one-time buffer alloc + // must be guarded, or we leak buffers. + mc = nd->ham_throttle.periodic_mc[nc_id]; + if (!mc) { + ret = mc_alloc_align(nd, MC_LIFESPAN_DEVICE, HAM_PERIODIC_NQ_BUF_SIZE, + 0, MEM_LOC_HOST, 0, nc_id, + NEURON_MEMALLOC_TYPE_NOTIFICATION_HOST, &mc); + if (ret) { + pr_err("Failed to allocate periodic throttle NQ buffer for NC %d: %d\n", + nc_id, ret); + neuron_ham_throttle_destroy(nd); + return ret; + } + nd->ham_throttle.periodic_mc[nc_id] = mc; + } + if (mc->va) + memset(mc->va, 0, HAM_PERIODIC_NQ_BUF_SIZE); + ndhal->ndhal_nq.nnq_set_hwaddr(nd, nc_id, HAM_PERIODIC_NQ_SLOT, + NQ_TYPE_THROTTLE, HAM_PERIODIC_NQ_BUF_SIZE, + mc->pa); + if (ham_throttle_program_nc(nd, nc_id)) + pr_warn("nd%d nc%d: failed to set periodic throttle routing\n", + nd->device_index, nc_id); + } + + return 0; +} + +void neuron_ham_throttle_destroy(struct neuron_device *nd) +{ + u8 nc_id; + + for (nc_id = 0; nc_id < ndhal->ndhal_address_map.nc_per_device; nc_id++) { + if (((1 << nc_id) & ndhal->ndhal_address_map.dev_nc_map) == 0) + continue; + + if (nd->ham_throttle.periodic_mc[nc_id]) { + ndhal->ndhal_nq.nnq_set_hwaddr(nd, nc_id, HAM_PERIODIC_NQ_SLOT, + NQ_TYPE_THROTTLE, 0, 0); + mc_free(&nd->ham_throttle.periodic_mc[nc_id]); + } + } +} + diff --git a/neuron_ham_throttle.h b/neuron_ham_throttle.h new file mode 100644 index 0000000..9faf029 --- /dev/null +++ b/neuron_ham_throttle.h @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + */ +#ifndef _NEURON_HAM_THROTTLE_H +#define _NEURON_HAM_THROTTLE_H + +#include + +struct neuron_device; + +/** + * neuron_ham_throttle_init() - Set up periodic throttle notification queues + * + * @nd: neuron device + * @return: 0 on success, negative on failure + */ +int neuron_ham_throttle_init(struct neuron_device *nd); + +/** + * neuron_ham_throttle_format() - format the aggregated periodic throttle state + * for one NC into @buf. + * + * @nd: neuron device + * @nc_id: neuron core / TPB index + * @buf: sysfs output buffer + * @len: size of @buf + * @return: number of bytes written + */ +ssize_t neuron_ham_throttle_format(struct neuron_device *nd, int nc_id, char *buf, size_t len); + +/** + * neuron_ham_throttle_set_aggregate_count() - set how many recent entries a + * subsequent read aggregates for one NC. + * + * @nd: neuron device + * @nc_id: neuron core / TPB index + * @buf: sysfs input buffer (decimal or 0x-prefixed count) + * @size: size of @buf + * @return: @size on success, negative errno on parse error + */ +ssize_t neuron_ham_throttle_set_aggregate_count(struct neuron_device *nd, int nc_id, + const char *buf, size_t size); + +/** + * neuron_ham_throttle_reprogram_nc() - Re-apply NOTIFIC routing for one NC. + * + * @nd: neuron device + * @nc_id: neuron core / TPB index + */ +void neuron_ham_throttle_reprogram_nc(struct neuron_device *nd, u8 nc_id); + +/** + * neuron_ham_throttle_destroy() - Tear down periodic throttle notification queues + * + * @nd: neuron device + */ +void neuron_ham_throttle_destroy(struct neuron_device *nd); + +#endif /* _NEURON_HAM_THROTTLE_H */ + diff --git a/neuron_ioctl.h b/neuron_ioctl.h index 7b42bc7..e246a31 100644 --- a/neuron_ioctl.h +++ b/neuron_ioctl.h @@ -147,6 +147,17 @@ struct neuron_ioctl_mem_copy64 { void *context; // [in] Opaque context pointer passed back in the completion queue. }; +struct neuron_ioctl_mem_copy64q { + __u64 src_mem_handle; // [in] Source memory handle from where data is copied. + __u64 dst_mem_handle; // [in] Destination memory handle to data is to be copied. + __u64 size; // [in] Size of the transfer. + __u64 src_offset; // [in] Offset in the source memory handle. + __u64 dst_offset; // [in] Offset in the destination memory handle. + __u64 sequence_num; // [in] The sequence number that uniquely identifies each async I/O. + void *context; // [in] Opaque context pointer passed back in the completion queue. + __u32 qid; // [in] Qid of the target queue. +}; + // Old drivers use the same mem copy ioctl number with this smaller payload. struct neuron_ioctl_mem_copy64_deprecated { __u64 src_mem_handle; // [in] Source memory handle from where data is copied. @@ -397,7 +408,7 @@ struct neuron_ioctl_notifications_init_v2 { __u32 nq_dev_id; // [in] Notification device Index __u32 nq_dev_type; // [in] Notification device type. __u32 nq_type; // [in] Notification queue type - __u32 engine_index; // [in] Engine Index. + __u32 engine_index; // [in] Engine Index; for NQ_DEVICE_TYPE_H2D_DMA the caller's owned physical NC (the driver selects that SEngine's top-level engine) __u32 size; // [in] Notification queue size in bytes __u32 on_host_memory; // [in] If true allocates NQ in host memory; else allocates in device memory __u32 hbm_index; // [in] HBM index within a device @@ -410,7 +421,7 @@ struct neuron_ioctl_notifications_init_with_realloc_v2 { __u32 nq_dev_id; // [in] Notification device Index __u32 nq_dev_type; // [in] Notification device type. __u32 nq_type; // [in] Notification queue type - __u32 engine_index; // [in] Engine Index. + __u32 engine_index; // [in] Engine Index; for NQ_DEVICE_TYPE_H2D_DMA the caller's owned physical NC (the driver selects that SEngine's top-level engine) __u32 size; // [in] Notification queue size in bytes __u32 on_host_memory; // [in] If true allocates NQ in host memory; else allocates in device memory __u32 hbm_index; // [in] HBM index within a device @@ -717,6 +728,29 @@ struct neuron_ioctl_host_mem_unpin { __u64 va; // [in] VA to unpin (must match exact VA from pin) }; +/** + * Queue and execute a caller-built pair of H2D DMA descriptors. + * + * Runtime supplies fully-built tx (m2s) and rx (s2m) descriptors; the driver + * copies them into the target H2T ring, appends its own completion descriptor + * pair, and drives the transfer through the shared queued-request pipeline. + * + * sequence_num == 0 selects the sync path: the ioctl blocks until completion + * and returns the transfer status. Non-zero selects the async path: the ioctl + * returns immediately and a completion queue entry with the caller's + * sequence_num and context is posted when the transfer completes. + */ +struct neuron_ioctl_h2d_desc_execute { + __u64 tx_descs; // [in] tx descriptors to queue up + __u64 rx_descs; // [in] rx descriptors to queue up + __u64 tx_descs_bytes; // [in] size of the tx_descs buffer (multiple of sizeof(union udma_desc)) + __u64 rx_descs_bytes; // [in] size of the rx_descs buffer + __s32 h2t_qid; // [in] h2t queue to use for the transfer. -1 = use default + __u32 nc_id; // [in] NeuronCore id that owns the target queue + __u64 sequence_num; // [in] 0 = sync, non-zero = async CQE sequence number + void *context; // [in] opaque context pointer echoed in the CQE (async only) +}; + #define NEURON_IOCTL_MAX_BAR_ENTRIES 16 struct neuron_ioctl_get_bar_info { __u32 query_type; // [in] 1=INTER_SERVER, 2=INTRA_SERVER @@ -725,6 +759,11 @@ struct neuron_ioctl_get_bar_info { __u64 bars[NEURON_IOCTL_MAX_BAR_ENTRIES]; // [out] 64-bit PCIe BAR address }; +struct neuron_ioctl_pcie_profile { + __u8 ctrl; // [in] control 0 = set; 1 = get + __u8 profile; // [in/out] pcie profile +}; + #define NEURON_IOCTL_BASE 'N' /* Deprecated reset related IOCTLs. Now it would always return success. */ @@ -758,6 +797,7 @@ struct neuron_ioctl_get_bar_info { #define NEURON_IOCTL_MEM_COPY _IOR(NEURON_IOCTL_BASE, 23, struct neuron_ioctl_mem_copy *) #define NEURON_IOCTL_MEM_COPY64_DEPRECATED _IOR(NEURON_IOCTL_BASE, 23, struct neuron_ioctl_mem_copy64_deprecated) #define NEURON_IOCTL_MEM_COPY64 _IOR(NEURON_IOCTL_BASE, 23, struct neuron_ioctl_mem_copy64) +#define NEURON_IOCTL_MEM_COPY64Q _IOR(NEURON_IOCTL_BASE, 23, struct neuron_ioctl_mem_copy64q) /** Copy data from/to given host buffer to/from memory_handle. (using DMA)*/ #define NEURON_IOCTL_MEM_BUF_COPY _IOWR(NEURON_IOCTL_BASE, 24, struct neuron_ioctl_mem_buf_copy *) @@ -958,4 +998,10 @@ struct neuron_ioctl_get_bar_info { /** Get switch fabric BAR info for a device */ #define NEURON_IOCTL_GET_BAR_INFO _IOWR(NEURON_IOCTL_BASE, 139, struct neuron_ioctl_get_bar_info) +/** Queue and execute pre-built H2D DMA descriptors */ +#define NEURON_IOCTL_H2D_DESC_EXECUTE _IOWR(NEURON_IOCTL_BASE, 140, struct neuron_ioctl_h2d_desc_execute) + +#define NEURON_IOCTL_PCIE_PROFILE _IOWR(NEURON_IOCTL_BASE, 141, struct neuron_ioctl_pcie_profile) + #endif + diff --git a/neuron_log.c b/neuron_log.c index b0a4f2d..f79e28f 100644 --- a/neuron_log.c +++ b/neuron_log.c @@ -160,3 +160,4 @@ int neuron_log_dump(struct neuron_device *nd, pid_t pid, uint32_t log_dump_limit } return 0; } + diff --git a/neuron_log.h b/neuron_log.h index 5645188..90325f2 100644 --- a/neuron_log.h +++ b/neuron_log.h @@ -75,3 +75,4 @@ void neuron_log_rec_add(struct neuron_device *nd, enum neuron_log_type type, uin int neuron_log_dump(struct neuron_device *nd, pid_t pid, uint32_t log_dump_limit); #endif + diff --git a/neuron_mc_handle.c b/neuron_mc_handle.c index 9ffbfba..67140e9 100644 --- a/neuron_mc_handle.c +++ b/neuron_mc_handle.c @@ -266,3 +266,4 @@ void nmch_handle_cleanup(struct neuron_device *nd) } nmch_service_set_down(nd); } + diff --git a/neuron_mc_handle.h b/neuron_mc_handle.h index 86e07c3..692f24d 100644 --- a/neuron_mc_handle.h +++ b/neuron_mc_handle.h @@ -84,3 +84,4 @@ int nmch_handle_init(struct neuron_device *nd); void nmch_handle_cleanup(struct neuron_device *nd); #endif + diff --git a/neuron_mempool.c b/neuron_mempool.c index f296fe7..07ac6e3 100644 --- a/neuron_mempool.c +++ b/neuron_mempool.c @@ -313,8 +313,6 @@ static void mp_destroy_gen_pool(struct neuron_mempool *mp) } } -// Upper 16MB is used internally by the firmware, don't use it in the allocation pool -#define MEMPOOL_CARVEOUT_SIZE 0x1000000 // 16MB /** * mpset_block_carveout_regions() * @@ -344,7 +342,9 @@ static int mpset_block_carveout_regions(struct neuron_device *nd, struct neuron_ for (hbm_index = 0; hbm_index < mpset->num_hbms; hbm_index++) { const dma_addr_t start_addr = device_dram_addr[hbm_index]; struct mem_chunk *mc = NULL; - u32 nc_id = hbm_index; // TODO is this right? + // TODO: nc_id picks the H2D DMA engine/queue for this mc - fix before + // the fw layout reader (neuron_fw_layout.c) reads the carveout by DMA + u32 nc_id = hbm_index; ret = mc_alloc_align(nd, MC_LIFESPAN_DEVICE, MEMPOOL_CARVEOUT_SIZE, 0, MEM_LOC_DEVICE, hbm_index, nc_id, NEURON_MEMALLOC_TYPE_NCDEV_DEVICE, &mc); if (ret) { pr_err("failed to allocate hbm carveout: ret=%d\n", ret); @@ -928,3 +928,4 @@ int mc_dump_all_chunks(struct neuron_device *nd, u32 hbm_index, u32 num_entries_ *num_entries_out = cnt; return 0; } + diff --git a/neuron_mempool.h b/neuron_mempool.h index 44be8fd..99220d8 100644 --- a/neuron_mempool.h +++ b/neuron_mempool.h @@ -65,7 +65,10 @@ struct neuron_mempool { u64 scratchpad_size; // only used for allocations of type NEURON_MEMALLOC_TYPE_CONTIGUOUS_SCRATCHPAD_DEVICE }; -#define MAX_NUM_HBMS 4 // max supported number of HBMs per device, VX_NUM_HBMS macros define it per specific chip +#define MAX_NUM_HBMS 4 // max supported number of HBMs per device, VX_NUM_HBMS macros define it per specific chip + +// First 16MB of every HBM is used internally by the firmware, don't use it in the allocation pool +#define MEMPOOL_CARVEOUT_SIZE 0x1000000 // 16MB // start page size for host MP #define MP_HOST_PAGE_SIZE_MIN (256UL * 1024) @@ -230,3 +233,4 @@ static inline bool mc_access_is_within_bounds(const struct mem_chunk *mc, u64 ac } #endif + diff --git a/neuron_metrics.c b/neuron_metrics.c index 9b65153..dffe9b0 100644 --- a/neuron_metrics.c +++ b/neuron_metrics.c @@ -1014,6 +1014,7 @@ static void nmetric_sample_high_freq(struct neuron_device *nd) { npower_sample_utilization(nd); nsysfsmetric_health_status_tick(nd); + fw_layout_poll(nd); } static void nmetric_aggregate_and_post_tick(struct neuron_device *nd, struct nmetric_versions *component_versions, u64 *curr_feature_bitmap, u64 *freed_feature_bitmap, u64 *const_u64_metrics, u64 *freed_const_u64_metrics, u8 tick) @@ -1244,3 +1245,4 @@ void nmetric_set_performance_profile(struct neuron_device *nd, int profile) { snprintf(nmetric_constant_metrics[NMETRIC_PROFILE_ID_IDX], NEURON_METRICS_VERSION_STRING_MAX_LEN + 1, "%d", ndhal->ndhal_perf.current_performance_profile); } + diff --git a/neuron_metrics.h b/neuron_metrics.h index 59585ad..1465ed9 100644 --- a/neuron_metrics.h +++ b/neuron_metrics.h @@ -216,3 +216,4 @@ void nmetric_set_performance_profile(struct neuron_device *nd, int profile); */ void nmetric_set_mode(struct neuron_device *nd, enum neuron_metrics_mode mode); #endif + diff --git a/neuron_mmap.c b/neuron_mmap.c index 0106332..95fbb40 100644 --- a/neuron_mmap.c +++ b/neuron_mmap.c @@ -543,3 +543,4 @@ unsigned long nmmap_get_unmapped_area(struct file *filep, unsigned long addr, return ALIGN(aligned, NEURON_P2P_HUGE_PAGE_SZ); } + diff --git a/neuron_mmap.h b/neuron_mmap.h index 50c0dce..9e561e7 100644 --- a/neuron_mmap.h +++ b/neuron_mmap.h @@ -188,3 +188,4 @@ unsigned long nmmap_get_unmapped_area(struct file *filep, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); #endif + diff --git a/neuron_module.c b/neuron_module.c index aa56d91..abd74cd 100644 --- a/neuron_module.c +++ b/neuron_module.c @@ -20,13 +20,13 @@ #include "neuron_dma.h" #include "neuron_test.h" -MODULE_DESCRIPTION("Neuron Driver, built from SHA: 40a502a5d7e0f165b7e4d89588eb1da01c068e8e"); +MODULE_DESCRIPTION("Neuron Driver"); MODULE_LICENSE("GPL"); -MODULE_VERSION("2.30.2.0"); +MODULE_VERSION("2.31.7.0"); MODULE_ALIAS("pci:v00001d0fd00007064sv*sd*bc*sc*i*"); -const char driver_version[] = "2.30.2.0"; -const char driver_revision[] = "40a502a5d7e0f165b7e4d89588eb1da01c068e8e"; +const char driver_version[] = "2.31.7.0"; +const char driver_revision[] = "63b8760cad44a0c5c428065e92595d88203b7f51"; #ifdef CONFIG_FAULT_INJECTION @@ -102,3 +102,4 @@ static void __exit neuron_module_exit(void) module_init(neuron_module_init); module_exit(neuron_module_exit); + diff --git a/neuron_nq.c b/neuron_nq.c index 5dc1f26..6cea949 100644 --- a/neuron_nq.c +++ b/neuron_nq.c @@ -180,6 +180,45 @@ void nnq_destroy_nc(struct neuron_device *nd, u8 nc_id) } } +static void nnq_h2d_destroy(struct neuron_device *nd, pid_t owner) +{ + int h2d_nqid; + if (nd == NULL) + return; + + mutex_lock(&nd->lock); + // assert nqs are quiesced before freeing memchunks + // quiesced by writing disable reg and reading back value + for (h2d_nqid = 0; h2d_nqid < NUM_H2D_DMA_PER_DEVICE; h2d_nqid++) { + if (nd->h2d_nq_mc[h2d_nqid] == NULL) + continue; + if (owner != 0 && nd->h2d_nq_mc[h2d_nqid]->pid != owner) + continue; + + uint32_t h2d_engine_id = ndhal->ndhal_nq.nnq_get_h2d_id(h2d_nqid); + ndhal->ndhal_nq.nnq_set_nq_h2d_enable(nd, h2d_engine_id, NOTIFIC_NQ_ENABLE_RESET_VALUE); + ndhal->ndhal_nq.nnq_set_h2d_hwaddr(nd, h2d_engine_id, 0, 0); + } + + for (h2d_nqid = 0; h2d_nqid < NUM_H2D_DMA_PER_DEVICE; h2d_nqid++) { + if (nd->h2d_nq_mc[h2d_nqid] == NULL) + continue; + if (owner != 0 && nd->h2d_nq_mc[h2d_nqid]->pid != owner) + continue; + + uint32_t h2d_engine_id = ndhal->ndhal_nq.nnq_get_h2d_id(h2d_nqid); + u32 enable_val = 0xdeadbeef; + ndhal->ndhal_nq.nnq_get_nq_h2d_enable(nd, h2d_engine_id, &enable_val); + if (enable_val != NOTIFIC_NQ_ENABLE_RESET_VALUE) { + pr_warn("Failed to disable H2D Notification Queue %u (enable=0x%x). Cleaning Notification queue without disabling", h2d_engine_id, enable_val); + } + mc_free(&nd->h2d_nq_mc[h2d_nqid]); + nd->h2d_nq_mc[h2d_nqid] = NULL; + } + mutex_unlock(&nd->lock); + +} + void nnq_destroy_all(struct neuron_device *nd) { u8 nc_id; @@ -190,4 +229,74 @@ void nnq_destroy_all(struct neuron_device *nd) } nnq_destroy_nc(nd, nc_id); } + nnq_h2d_destroy(nd, 0); +} + +int nnq_h2d_init(struct neuron_device *nd, uint32_t h2d_eng_id, u32 nc_id, + u32 size, u32 on_host_memory, u32 hbm_index, + bool force_alloc_mem, struct mem_chunk **nq_mc, u64 *mmap_offset) +{ + int h2d_nqid, ret; + struct mem_chunk *mc; + pid_t owner = task_tgid_nr(current); + + if (size & (size - 1)) { + pr_err("notification ring size must be power of 2\n"); + return -EINVAL; + } + + if (!ndhal->ndhal_nq.nnq_set_h2d_hwaddr) + return -ENOSYS; + + h2d_nqid = ndhal->ndhal_nq.nnq_get_nq_h2d_id(h2d_eng_id); + if (h2d_nqid < 0 || h2d_nqid >= NUM_H2D_DMA_PER_DEVICE) + return -EINVAL; + + mutex_lock(&nd->lock); + + if (nd->h2d_nq_mc[h2d_nqid] != NULL && nd->h2d_nq_mc[h2d_nqid]->pid != owner) { + mutex_unlock(&nd->lock); + return -EBUSY; + } + + mc = nd->h2d_nq_mc[h2d_nqid]; + if (mc == NULL || force_alloc_mem) { + struct mem_chunk *_mc = NULL; + + ret = mc_alloc_align(nd, MC_LIFESPAN_CUR_PROCESS, size, on_host_memory ? 0 : size, + on_host_memory ? MEM_LOC_HOST : MEM_LOC_DEVICE, + hbm_index, nc_id, + on_host_memory ? NEURON_MEMALLOC_TYPE_NOTIFICATION_HOST + : NEURON_MEMALLOC_TYPE_NOTIFICATION_DEVICE, + &_mc); + if (ret) { + mutex_unlock(&nd->lock); + return ret; + } + + ret = ndhal->ndhal_nq.nnq_set_h2d_hwaddr(nd, h2d_eng_id, size, _mc->pa); + if (ret) { + mc_free(&_mc); + mutex_unlock(&nd->lock); + return ret; + } + + if (mc) + mc_free(&mc); + nd->h2d_nq_mc[h2d_nqid] = _mc; + mc = _mc; + } + + *mmap_offset = (mc->mem_location == MEM_LOC_HOST) ? nmmap_offset(mc) : (u64)-1; + *nq_mc = mc; + + mutex_unlock(&nd->lock); + return 0; +} + +void nnq_h2d_handle_process_exit(struct neuron_device *nd) +{ + pid_t owner = task_tgid_nr(current); + nnq_h2d_destroy(nd, owner); } + diff --git a/neuron_nq.h b/neuron_nq.h index 6292ff3..989b426 100644 --- a/neuron_nq.h +++ b/neuron_nq.h @@ -50,6 +50,25 @@ static inline void notific_write_nq_f_size(void __iomem *base, size_t index, reg_write32(base + offset, value); } +#define NOTIFIC_NQ_ENABLE_OFFSET_START 0x14 +#define NOTIFIC_NQ_ENABLE_OFFSET NOTIFIC_NQ_ENABLE_OFFSET_START +#define NOTIFIC_NQ_ENABLE_RESET_VALUE 0 +static inline void notific_write_nq_enable(void __iomem *base, uint32_t value) +{ + const size_t offset = NOTIFIC_NQ_ENABLE_OFFSET; + + reg_write32(base + offset, value); +} + +static inline u32 notific_read_nq_enable(void __iomem *base) +{ + const size_t offset = NOTIFIC_NQ_ENABLE_OFFSET; + u32 value = 0; + reg_read32(base + offset, &value); + + return value; +} + #define NOTIFIC_NQ_HEAD_OFFSET 0x10c /** @@ -106,4 +125,38 @@ void nnq_destroy_nc(struct neuron_device *nd, u8 nc_id); */ void nnq_destroy_all(struct neuron_device *nd); +/** + * nnq_init_h2d() - Enable a notification queue on an H2D DMA engine. + * + * Only one struct file may own notifications on a given H2D engine at a time; + * subsequent calls from a different file return -EBUSY. A re-init from the + * current owner is allowed (e.g. to grow the queue or change memory location). + * + * @nd: neuron device + * @h2d_eng_id: absolute H2D DMA engine id (must satisfy ndhal_ndmar.is_h2d_eng) + * @nc_id: physical NC that owns the buffer and is used for later buffer DMA + * @size: size of queue in bytes (must be a power of two) + * @on_host_memory: if true, NQ lives in host memory; else in device HBM + * @hbm_index: HBM channel to use when on_host_memory == false + * @force_alloc_mem: if true, drop any existing NQ memory and allocate fresh + * @nq_mc[out]: memchunk backing the queue + * @mmap_offset[out]: mmap offset for host-memory NQs, -1 otherwise + * + * Return: 0 on success, -EBUSY if owned by another file, other -errno on failure. + */ +int nnq_h2d_init(struct neuron_device *nd, uint32_t h2d_eng_id, u32 nc_id, + u32 size, u32 on_host_memory, u32 hbm_index, + bool force_alloc_mem, struct mem_chunk **nq_mc, u64 *mmap_offset); + +/** + * nnq_destroy_h2d_for_owner() - Release any H2D NQs held by @owner. + * + * Called on fd close to drop ownership of every H2D engine still attributed + * to @owner and free the backing NQ memory. + * + * @nd: neuron device + */ +void nnq_h2d_handle_process_exit(struct neuron_device *nd); + #endif + diff --git a/neuron_p2p.c b/neuron_p2p.c index 99e3c3f..2bd2fc7 100644 --- a/neuron_p2p.c +++ b/neuron_p2p.c @@ -174,3 +174,4 @@ int neuron_p2p_unregister_va(struct neuron_p2p_va_info *vainfo) return 0; } EXPORT_SYMBOL_GPL(neuron_p2p_unregister_va); + diff --git a/neuron_p2p.h b/neuron_p2p.h index 915886c..0f69e60 100644 --- a/neuron_p2p.h +++ b/neuron_p2p.h @@ -43,4 +43,4 @@ int neuron_p2p_register_va(u64 virtual_address, u64 length, struct neuron_p2p_va */ int neuron_p2p_unregister_va(struct neuron_p2p_va_info *vainfo); -#endif \ No newline at end of file +#endif diff --git a/neuron_pci.c b/neuron_pci.c index b1356c6..c9023bf 100644 --- a/neuron_pci.c +++ b/neuron_pci.c @@ -25,6 +25,7 @@ #include "neuron_pci.h" #include "neuron_log.h" #include "neuron_cdev.h" +#include "neuron_ham_throttle.h" static struct pci_device_id neuron_pci_dev_ids[] = { @@ -38,6 +39,19 @@ static struct pci_device_id neuron_pci_dev_ids[] = { }, }; +#define EFA_V1_DEVICE_ID 0xEFA0 +#define EFA_V2_DEVICE_ID 0xEFA1 +#define EFA_V3_DEVICE_ID 0xEFA2 +#define EFA_V4_DEVICE_ID 0xEFA3 + +static const struct pci_device_id neuron_efa_dev_ids[] = { + { PCI_DEVICE(AMZN_VENDOR_ID, EFA_V4_DEVICE_ID), .driver_data = NEURON_EFA_VERSION_V4 }, + { PCI_DEVICE(AMZN_VENDOR_ID, EFA_V3_DEVICE_ID), .driver_data = NEURON_EFA_VERSION_V3 }, + { PCI_DEVICE(AMZN_VENDOR_ID, EFA_V2_DEVICE_ID), .driver_data = NEURON_EFA_VERSION_V2 }, + { PCI_DEVICE(AMZN_VENDOR_ID, EFA_V1_DEVICE_ID), .driver_data = NEURON_EFA_VERSION_V1 }, + { 0, }, +}; + // some old kernels do not have pci_info defined. #ifndef pci_info #define pci_info(pdev, fmt, arg...) dev_info(&(pdev)->dev, fmt, ##arg) @@ -212,6 +226,32 @@ static int neuron_pci_device_close(struct neuron_device *nd) return 0; } +static enum neuron_efa_version neuron_pci_efa_version = NEURON_EFA_VERSION_NONE; + +enum neuron_efa_version neuron_pci_get_efa_version(void) +{ + return neuron_pci_efa_version; +} + +static void neuron_pci_probe_efa_version(void) +{ + const struct pci_device_id *id; + struct pci_dev *pdev; + + if (neuron_pci_efa_version != NEURON_EFA_VERSION_NONE) { + return; + } + + for (id = neuron_efa_dev_ids; id->vendor; id++) { + pdev = pci_get_device(id->vendor, id->device, NULL); + if (pdev) { + neuron_pci_efa_version = (enum neuron_efa_version)id->driver_data; + pci_dev_put(pdev); + return; + } + } +} + static void neuron_pci_set_device_architecture(struct neuron_device *nd) { unsigned short device = nd->pdev->device; @@ -397,6 +437,8 @@ static int neuron_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) goto fail_dhal_init; } + neuron_pci_probe_efa_version(); + // map apb bar ret = neuron_pci_reserve_bar(dev, ndhal->ndhal_pci.apb_bar, "APB"); if (ret) { @@ -427,6 +469,11 @@ static int neuron_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) goto fail_bar4_resource; } + ret = ndhal->ndhal_pci.neuron_pci_init(nd); + if (ret) { + goto fail_bar4_resource; + } + #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) nd->device_index = atomic_fetch_add(1, &device_count); @@ -465,6 +512,10 @@ static int neuron_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) if (ret) goto fail_memset_mc; + // initialize the firmware layout reader before the metrics thread that polls it. + // never fails - unsupported setups leave the reader disabled + fw_layout_init(nd); + if (nmetric_log_posts != 0) { // initialize metric aggregation and posting ret = nmetric_init(nd); @@ -481,6 +532,7 @@ static int neuron_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) fail_nmetric_resource: nmetric_stop_thread(nd); + fw_layout_destroy(nd); mc_free(&nd->memset_mc); fail_memset_mc: mc_free(&nd->ndma_q_dummy_mc); @@ -514,12 +566,15 @@ static void neuron_pci_remove(struct pci_dev *dev) if (nd == NULL) return; - ndma_h2d_stop_cmpltn_thread(nd); - nr_stop_thread(nd); + neuron_ham_throttle_destroy(nd); + nmetric_stop_thread(nd); + // after the metrics thread is stopped nothing polls the firmware layout + fw_layout_destroy(nd); + ndhal->ndhal_ext_cleanup(); neuron_pci_release_bar(dev, ndhal->ndhal_pci.apb_bar); @@ -573,3 +628,4 @@ void neuron_pci_module_exit(void) pci_unregister_driver(&neuron_pci_driver); neuron_dhal_free(); } + diff --git a/neuron_pci.h b/neuron_pci.h index afe7afe..94a03ad 100644 --- a/neuron_pci.h +++ b/neuron_pci.h @@ -12,6 +12,18 @@ #define BAR_UNUSED -1 +/* + * EFA (K2) generation paired with neuron devices in this system. Discovered + * once at first neuron-device probe; identical for all devices in a system. + */ +enum neuron_efa_version { + NEURON_EFA_VERSION_NONE = 0, /* no EFA present */ + NEURON_EFA_VERSION_V1, + NEURON_EFA_VERSION_V2, + NEURON_EFA_VERSION_V3, + NEURON_EFA_VERSION_V4, +}; + extern struct neuron_device *neuron_devices[MAX_NEURON_DEVICE_COUNT]; extern int total_neuron_devices; extern int wc_enable; @@ -32,6 +44,13 @@ struct neuron_device *neuron_pci_get_device(u8 device_index); */ int neuron_pci_handle_dup_routing_id(void); +/** + * neuron_pci_get_efa_version() - Get EFA (K2) card version + * + * Return: EFA card version if present, NEURON_EFA_VERSION_NONE if no EFA device is present + */ +enum neuron_efa_version neuron_pci_get_efa_version(void); + /** * neuron_pci_module_init() - Initialize Neuron PCI driver. * @@ -44,4 +63,4 @@ int neuron_pci_module_init(void); */ void neuron_pci_module_exit(void); -#endif \ No newline at end of file +#endif diff --git a/neuron_pid.c b/neuron_pid.c index 55e855f..3f5c1fe 100644 --- a/neuron_pid.c +++ b/neuron_pid.c @@ -148,3 +148,4 @@ int npid_get_allocated_memory(struct neuron_device *nd, pid_t pid, size_t *host_ *device_memory = nd->attached_processes[slot].memory_used[MEM_LOC_DEVICE - 1]; return 0; } + diff --git a/neuron_pid.h b/neuron_pid.h index 6f4bb9d..9fc4102 100644 --- a/neuron_pid.h +++ b/neuron_pid.h @@ -116,3 +116,4 @@ int npid_dec_allocated_memory(struct neuron_device *nd, enum mem_location locati int npid_get_allocated_memory(struct neuron_device *nd, pid_t pid, size_t *host_memory, size_t *device_memory); #endif + diff --git a/neuron_platform.c b/neuron_platform.c new file mode 100644 index 0000000..3aae56b --- /dev/null +++ b/neuron_platform.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + */ + +#include + +#include "neuron_arch.h" +#include "neuron_platform.h" + +const char *const trn1_nc_efa_bdf_tbl[32] = { + "0000:10:1a.0", "0000:10:1a.0", + "0000:20:19.0", "0000:20:19.0", + "0000:10:1a.0", "0000:10:1a.0", + "0000:20:19.0", "0000:20:19.0", + "0000:a0:1a.0", "0000:a0:1a.0", + "0000:90:19.0", "0000:90:19.0", + "0000:a0:1a.0", "0000:a0:1a.0", + "0000:90:19.0", "0000:90:19.0", + "0000:10:19.0", "0000:10:19.0", + "0000:20:1a.0", "0000:20:1a.0", + "0000:10:19.0", "0000:10:19.0", + "0000:20:1a.0", "0000:20:1a.0", + "0000:a0:19.0", "0000:a0:19.0", + "0000:90:1a.0", "0000:90:1a.0", + "0000:a0:19.0", "0000:a0:19.0", + "0000:90:1a.0", "0000:90:1a.0", +}; + +const char *const trn1n_nc_efa_bdf_tbl[32] = { + "0000:10:17.0", "0000:10:17.0", + "0000:20:19.0", "0000:20:19.0", + "0000:10:1a.0", "0000:10:1a.0", + "0000:20:18.0", "0000:20:18.0", + "0000:a0:1a.0", "0000:a0:1a.0", + "0000:90:18.0", "0000:90:18.0", + "0000:a0:17.0", "0000:a0:17.0", + "0000:90:19.0", "0000:90:19.0", + "0000:10:19.0", "0000:10:19.0", + "0000:20:17.0", "0000:20:17.0", + "0000:10:18.0", "0000:10:18.0", + "0000:20:1a.0", "0000:20:1a.0", + "0000:a0:18.0", "0000:a0:18.0", + "0000:90:1a.0", "0000:90:1a.0", + "0000:a0:19.0", "0000:a0:19.0", + "0000:90:17.0", "0000:90:17.0", +}; + +const char *const trn2_nc_efa_bdf_tbl[128] = { + "0000:c9:00.0", "0000:c9:00.0", "0000:c9:00.0", "0000:c9:00.0", "0000:c9:00.0", "0000:c9:00.0", "0000:c9:00.0", "0000:c9:00.0", + "0000:ca:00.0", "0000:ca:00.0", "0000:ca:00.0", "0000:ca:00.0", "0000:ca:00.0", "0000:ca:00.0", "0000:ca:00.0", "0000:ca:00.0", + "0000:6c:00.0", "0000:6c:00.0", "0000:6c:00.0", "0000:6c:00.0", "0000:6c:00.0", "0000:6c:00.0", "0000:6c:00.0", "0000:6c:00.0", + "0000:6d:00.0", "0000:6d:00.0", "0000:6d:00.0", "0000:6d:00.0", "0000:6d:00.0", "0000:6d:00.0", "0000:6d:00.0", "0000:6d:00.0", + "0000:f5:00.0", "0000:f5:00.0", "0000:f5:00.0", "0000:f5:00.0", "0000:f5:00.0", "0000:f5:00.0", "0000:f5:00.0", "0000:f5:00.0", + "0000:f6:00.0", "0000:f6:00.0", "0000:f6:00.0", "0000:f6:00.0", "0000:f6:00.0", "0000:f6:00.0", "0000:f6:00.0", "0000:f6:00.0", + "0000:98:00.0", "0000:98:00.0", "0000:98:00.0", "0000:98:00.0", "0000:98:00.0", "0000:98:00.0", "0000:98:00.0", "0000:98:00.0", + "0000:99:00.0", "0000:99:00.0", "0000:99:00.0", "0000:99:00.0", "0000:99:00.0", "0000:99:00.0", "0000:99:00.0", "0000:99:00.0", + "0000:b4:00.0", "0000:b4:00.0", "0000:b4:00.0", "0000:b4:00.0", "0000:b4:00.0", "0000:b4:00.0", "0000:b4:00.0", "0000:b4:00.0", + "0000:b3:00.0", "0000:b3:00.0", "0000:b3:00.0", "0000:b3:00.0", "0000:b3:00.0", "0000:b3:00.0", "0000:b3:00.0", "0000:b3:00.0", + "0000:57:00.0", "0000:57:00.0", "0000:57:00.0", "0000:57:00.0", "0000:57:00.0", "0000:57:00.0", "0000:57:00.0", "0000:57:00.0", + "0000:56:00.0", "0000:56:00.0", "0000:56:00.0", "0000:56:00.0", "0000:56:00.0", "0000:56:00.0", "0000:56:00.0", "0000:56:00.0", + "0000:e0:00.0", "0000:e0:00.0", "0000:e0:00.0", "0000:e0:00.0", "0000:e0:00.0", "0000:e0:00.0", "0000:e0:00.0", "0000:e0:00.0", + "0000:df:00.0", "0000:df:00.0", "0000:df:00.0", "0000:df:00.0", "0000:df:00.0", "0000:df:00.0", "0000:df:00.0", "0000:df:00.0", + "0000:83:00.0", "0000:83:00.0", "0000:83:00.0", "0000:83:00.0", "0000:83:00.0", "0000:83:00.0", "0000:83:00.0", "0000:83:00.0", + "0000:82:00.0", "0000:82:00.0", "0000:82:00.0", "0000:82:00.0", "0000:82:00.0", "0000:82:00.0", "0000:82:00.0", "0000:82:00.0", +}; + +const char *const trn2n_nc_efa_bdf_tbl[128] = { + "0000:c4:00.0", "0000:c4:00.0", "0000:c6:00.0", "0000:c6:00.0", "0000:c5:00.0", "0000:c5:00.0", "0000:c3:00.0", "0000:c3:00.0", + "0000:c8:00.0", "0000:c8:00.0", "0000:ca:00.0", "0000:ca:00.0", "0000:c9:00.0", "0000:c9:00.0", "0000:c7:00.0", "0000:c7:00.0", + "0000:67:00.0", "0000:67:00.0", "0000:69:00.0", "0000:69:00.0", "0000:68:00.0", "0000:68:00.0", "0000:66:00.0", "0000:66:00.0", + "0000:6b:00.0", "0000:6b:00.0", "0000:6d:00.0", "0000:6d:00.0", "0000:6c:00.0", "0000:6c:00.0", "0000:6a:00.0", "0000:6a:00.0", + "0000:f0:00.0", "0000:f0:00.0", "0000:f2:00.0", "0000:f2:00.0", "0000:f1:00.0", "0000:f1:00.0", "0000:ef:00.0", "0000:ef:00.0", + "0000:f4:00.0", "0000:f4:00.0", "0000:f6:00.0", "0000:f6:00.0", "0000:f5:00.0", "0000:f5:00.0", "0000:f3:00.0", "0000:f3:00.0", + "0000:93:00.0", "0000:93:00.0", "0000:95:00.0", "0000:95:00.0", "0000:94:00.0", "0000:94:00.0", "0000:92:00.0", "0000:92:00.0", + "0000:97:00.0", "0000:97:00.0", "0000:99:00.0", "0000:99:00.0", "0000:98:00.0", "0000:98:00.0", "0000:96:00.0", "0000:96:00.0", + "0000:b2:00.0", "0000:b2:00.0", "0000:b4:00.0", "0000:b4:00.0", "0000:b3:00.0", "0000:b3:00.0", "0000:b1:00.0", "0000:b1:00.0", + "0000:ae:00.0", "0000:ae:00.0", "0000:b0:00.0", "0000:b0:00.0", "0000:af:00.0", "0000:af:00.0", "0000:ad:00.0", "0000:ad:00.0", + "0000:55:00.0", "0000:55:00.0", "0000:57:00.0", "0000:57:00.0", "0000:56:00.0", "0000:56:00.0", "0000:54:00.0", "0000:54:00.0", + "0000:51:00.0", "0000:51:00.0", "0000:53:00.0", "0000:53:00.0", "0000:52:00.0", "0000:52:00.0", "0000:50:00.0", "0000:50:00.0", + "0000:de:00.0", "0000:de:00.0", "0000:e0:00.0", "0000:e0:00.0", "0000:df:00.0", "0000:df:00.0", "0000:dd:00.0", "0000:dd:00.0", + "0000:da:00.0", "0000:da:00.0", "0000:dc:00.0", "0000:dc:00.0", "0000:db:00.0", "0000:db:00.0", "0000:d9:00.0", "0000:d9:00.0", + "0000:81:00.0", "0000:81:00.0", "0000:83:00.0", "0000:83:00.0", "0000:82:00.0", "0000:82:00.0", "0000:80:00.0", "0000:80:00.0", + "0000:7d:00.0", "0000:7d:00.0", "0000:7f:00.0", "0000:7f:00.0", "0000:7e:00.0", "0000:7e:00.0", "0000:7c:00.0", "0000:7c:00.0", +}; + +const char *const trn3pd_nc_efa_bdf_tbl[128] = { + "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", + "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", + "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", + "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", + "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", + "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", + "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", + "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", + "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", + "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", + "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", + "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", + "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", + "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", + "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", + "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", +}; + +const char *const trn3pds_nc_efa_bdf_tbl[128] = { + "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", "0000:a9:00.0", + "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", "0000:a8:00.0", + "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", "0000:bf:00.0", + "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", "0000:be:00.0", + "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", "0000:d5:00.0", + "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", "0000:d4:00.0", + "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", "0000:eb:00.0", + "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", "0000:ea:00.0", + "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", "0000:4c:00.0", + "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", "0000:4b:00.0", + "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", "0000:62:00.0", + "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", "0000:61:00.0", + "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", "0000:78:00.0", + "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", "0000:77:00.0", + "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", "0000:8e:00.0", + "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", "0000:8d:00.0", +}; + +const struct neuron_platform_desc *nplatform_get_platform_desc(const struct neuron_platform_desc *table) +{ + const char *name; + char buf[128]; + int i, n; + + if (narch_get_instance_type_name(buf, sizeof(buf))) { + return NULL; + } + + for (i = 0; table[i].instance_names != NULL; i++) { + for (n = 0; table[i].instance_names[n] != NULL; n++) { + name = table[i].instance_names[n]; + if (strncmp(buf, name, strlen(name)) == 0) { + return &table[i]; + } + } + } + + return NULL; +} + diff --git a/neuron_platform.h b/neuron_platform.h new file mode 100644 index 0000000..798a69d --- /dev/null +++ b/neuron_platform.h @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + */ + +#ifndef NEURON_PLATFORM_H +#define NEURON_PLATFORM_H + +#include +#include +#include + +#include "share/neuron_driver_shared.h" + +struct neuron_device; + +enum neuron_platform_type { + NEURON_PLATFORM_TYPE_STD = 0, + NEURON_PLATFORM_TYPE_ULTRASERVER = 1, + NEURON_PLATFORM_TYPE_PDS = 2, + NEURON_PLATFORM_TYPE_MAX = 3, + NEURON_PLATFORM_TYPE_INVALID, +}; + +enum neuron_platform_operation_type { + NEURON_PLATFORM_OP_TYPE_DEVOPEN = 0, + NEURON_PLATFORM_OP_TYPE_EXEC = 1, +}; + +enum neuron_platform_routing_type { + NEURON_PLATFORM_ROUTING_TYPE_INVALID = 0, + NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL, + NEURON_PLATFORM_ROUTING_TYPE_TABLE, +}; + +enum neuron_pod_discovery_type { + NEURON_POD_DISCOVERY_TYPE_NONE = 0, + NEURON_POD_DISCOVERY_TYPE_ELECTION, + NEURON_POD_DISCOVERY_TYPE_FW_IO, +}; + +enum neuron_platform_family { + NEURON_PLATFORM_FAMILY_NONE = 0, + NEURON_PLATFORM_FAMILY_INF2, + NEURON_PLATFORM_FAMILY_TRN1, + NEURON_PLATFORM_FAMILY_TRN1N, + NEURON_PLATFORM_FAMILY_TRN2, + NEURON_PLATFORM_FAMILY_TRN2_ULTRASERVER, + NEURON_PLATFORM_FAMILY_TRN3_GEN1_ULTRASERVER, + NEURON_PLATFORM_FAMILY_TRN3_GEN2_ULTRASERVER, +}; + +enum neuron_soc_sub_version { + NEURON_SOC_SUB_VERSION_A0 = 0, + NEURON_SOC_SUB_VERSION_B1 = 1, +}; + +enum neuron_intra_rack_topology { + NEURON_INTRA_RACK_TOPOLOGY_NONE = 0, + NEURON_INTRA_RACK_TOPOLOGY_RING, + NEURON_INTRA_RACK_TOPOLOGY_TORUS, + NEURON_INTRA_RACK_TOPOLOGY_SWITCH, +}; + +enum neuron_inter_rack_topology { + NEURON_INTER_RACK_TOPOLOGY_NONE = 0, + NEURON_INTER_RACK_TOPOLOGY_P2P, +}; + +enum neuron_switch_type { + NEURON_SWITCH_TYPE_NONE = 0, + NEURON_SWITCH_TYPE_V1, + NEURON_SWITCH_TYPE_V2, +}; + +struct neuron_intra_rack_interconnect { + enum neuron_intra_rack_topology topology; + enum neuron_switch_type switch_type; +}; + +struct neuron_inter_rack_interconnect { + enum neuron_inter_rack_topology topology; + enum neuron_switch_type switch_type; +}; + +struct neuron_interconnect_info { + struct neuron_intra_rack_interconnect intra_rack; + struct neuron_inter_rack_interconnect inter_rack; +}; + +struct neuron_efa_bdf_map { + const char *const *bdfs; + u16 count; +}; + +struct neuron_platform_routing_config { + enum neuron_platform_routing_type type; + union { + struct { + const u32 *map; + u32 size; + } table; + }; +}; + +struct neuron_platform_desc { + const char *const *instance_names; + enum neuron_platform_type type; + enum neuron_platform_family family; + + struct neuron_platform_routing_config routing_config; + u8 device_count; + u8 default_node_cnt; + enum neuron_pod_type pod_type; + enum neuron_pod_discovery_type pod_discovery; + u8 max_bars; + struct neuron_interconnect_info interconnect; + struct neuron_efa_bdf_map nc_efa_bdf_tbl; + u8 num_sleds_per_server; + u8 num_chips_per_sled; +}; + +extern const char *const trn1_nc_efa_bdf_tbl[32]; +extern const char *const trn1n_nc_efa_bdf_tbl[32]; +extern const char *const trn2_nc_efa_bdf_tbl[128]; +extern const char *const trn2n_nc_efa_bdf_tbl[128]; +extern const char *const trn3pd_nc_efa_bdf_tbl[128]; +extern const char *const trn3pds_nc_efa_bdf_tbl[128]; + +/** + * nplatform_get_platform_desc() - Look up the platform descriptor for this instance. + * + * @table: Per-arch table of platform descriptors + * + * Return: Matching platform descriptor + */ +const struct neuron_platform_desc *nplatform_get_platform_desc(const struct neuron_platform_desc *table); + +extern const struct attribute_group neuron_soc_group; +extern const struct attribute_group efa_group; +extern const struct attribute_group platform_group; + +/** + * nplatform_sysfs_add_efa_symlink() - Create neuron_coreN/info/efa_device symlink to the paired EFA PCI device. + * + * @nd: Neuron device the neuron-core belongs to. + * @nc_kobj: Kobject of the neuron_coreN/info directory to attach the symlink to. + * @nc_id: Neuron-core index within the device. + * + * Return: 0 on success or when the platform doesn't have efa support + * Negative errno on failure: + * -ENOENT: missing BDF entry + * -EINVAL: malformed BDF string + * -ENODEV: EFA PCI device not enumerated + * or the return value from sysfs_create_link(). + */ +int nplatform_sysfs_add_efa_symlink(struct neuron_device *nd, struct kobject *nc_kobj, int nc_id); + +#endif // NEURON_PLATFORM_H + diff --git a/neuron_power.c b/neuron_power.c index 6613ec7..c0cca36 100644 --- a/neuron_power.c +++ b/neuron_power.c @@ -624,3 +624,4 @@ int npower_init_stats(struct neuron_device *nd) return 0; } + diff --git a/neuron_power.h b/neuron_power.h index daed6fe..93a93e0 100644 --- a/neuron_power.h +++ b/neuron_power.h @@ -189,4 +189,4 @@ int npower_init_stats(struct neuron_device *nd); */ bool npower_enabled_in_fw(struct neuron_device *nd); -#endif // _NEURON_POWER_H \ No newline at end of file +#endif // _NEURON_POWER_H diff --git a/neuron_reg_access.c b/neuron_reg_access.c index eceac3e..897951b 100644 --- a/neuron_reg_access.c +++ b/neuron_reg_access.c @@ -30,3 +30,4 @@ inline int reg_write32_masked(u32 __iomem *addr, u32 mask, u32 data) reg_write32(addr, MASK_VAL(mask, data, temp)); return 0; } + diff --git a/neuron_reg_access.h b/neuron_reg_access.h index 0b2be7a..a21f311 100644 --- a/neuron_reg_access.h +++ b/neuron_reg_access.h @@ -43,3 +43,4 @@ void reg_write32(u32 __iomem *addr, u32 value); int reg_write32_masked(u32 __iomem *addr, u32 mask, u32 data); #endif + diff --git a/neuron_reset.c b/neuron_reset.c index 7f01c51..e5aed6f 100644 --- a/neuron_reset.c +++ b/neuron_reset.c @@ -18,6 +18,7 @@ #include "neuron_ioctl.h" #include "neuron_device.h" #include "neuron_fw_io.h" +#include "neuron_ham_throttle.h" #include "neuron_dhal.h" #include "neuron_nq.h" #include "neuron_test.h" @@ -174,6 +175,7 @@ static int nr_reset_thread_fn(void *arg) // This path is taken by internal driver reset logic, there is no need to move // the request to the completion queue, since waiters will be polling device state instead if (state == NEURON_RESET_STATE_COMPLETED) { + neuron_ham_throttle_init(nd); nd->device_state = NEURON_DEVICE_STATE_READY; } else { nd->device_state = NEURON_DEVICE_STATE_INVALID; @@ -270,6 +272,7 @@ int nr_start_ncs(struct neuron_device *nd, uint32_t nc_map, uint32_t request_id) ndmar_init_ncs(nd, nc_map); nr_call_post_reset_config(nd, nc_map, true); if (request_id == NEURON_RESET_REQUEST_ALL) { + neuron_ham_throttle_init(nd); nd->device_state = NEURON_DEVICE_STATE_READY; } return 0; @@ -456,3 +459,4 @@ int nr_initiate_reset_via_fw(struct neuron_device *nd, uint32_t nc_map, uint32_t nmetric_increment_reset_failure_count(nd, is_device_reset); // Record the reset failure in metrics return -ETIMEDOUT; } + diff --git a/neuron_reset.h b/neuron_reset.h index 43a9025..d2cd0d0 100644 --- a/neuron_reset.h +++ b/neuron_reset.h @@ -129,3 +129,4 @@ int nr_initiate_reset_via_fw(struct neuron_device *nd, uint32_t nc_map, uint32_t int nr_msleep_stoppable(struct neuron_device *nd, uint32_t msec); #endif + diff --git a/neuron_ring.c b/neuron_ring.c index 4579416..f7d1009 100644 --- a/neuron_ring.c +++ b/neuron_ring.c @@ -18,11 +18,11 @@ #include "neuron_dhal.h" #include "neuron_ring.h" -int nc_per_dev_param = 1; +int nc_per_dev_param = 0; module_param(nc_per_dev_param, int, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); MODULE_PARM_DESC(nc_per_dev_param, "Number of neuron cores"); -int dev_nc_map = 1; +int dev_nc_map = 0; module_param(dev_nc_map, int, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); MODULE_PARM_DESC(dev_nc_map, "Map of active neuron cores"); @@ -387,7 +387,7 @@ static int ndmar_h2t_ring_alloc(struct neuron_device *nd, int nc_id, int qid) goto error; } - ret = ndma_ctx_queue_init(&ring->dma_ctx_queue); + ret = ndma_zca_dma_ioq_init(eng, ring, &ring->h2t_dma_ioq); if (ret) { pr_err("can't initialize dma context queue\n"); goto error_ctx_queue; @@ -579,6 +579,10 @@ int ndmar_eng_get_state(struct neuron_device *nd, int eng_id, struct neuron_dma_ return -EINVAL; udma = &eng->udma; + if (udma->udma_regs_m2s == NULL || udma->udma_regs_s2m == NULL) { + ndmar_release_engine(eng); + return -EINVAL; + } state->tx_state = udma_state_get(udma, UDMA_TX); state->rx_state = udma_state_get(udma, UDMA_RX); @@ -772,6 +776,8 @@ int ndmar_init(struct neuron_device *nd) static void ndmar_h2t_ring_free(struct ndma_eng *eng, struct ndma_ring *ring) { + ndma_zca_dma_ioq_free(&ring->h2t_dma_ioq); + if (ring->tx_mc) { mc_free(&ring->tx_mc); ring->tx_mc = NULL; @@ -792,7 +798,6 @@ static void ndmar_h2t_ring_free(struct ndma_eng *eng, struct ndma_ring *ring) ring->h2t_completion_mc = NULL; } - ndma_ctx_queue_free(eng, ring, &ring->dma_ctx_queue); ndma_h2d_compl_queue_destroy(&ring->dma_compl_queue); ndmar_h2t_ring_state_clr(ring); @@ -865,4 +870,6 @@ void ndmar_close_ncs(struct neuron_device *nd, uint32_t nc_map) void ndmar_close(struct neuron_device *nd) { ndmar_close_ncs(nd, NEURON_NC_MAP_DEVICE); + ndma_h2d_stop_cmpltn_thread(nd); } + diff --git a/neuron_ring.h b/neuron_ring.h index e99ea40..18ed15d 100644 --- a/neuron_ring.h +++ b/neuron_ring.h @@ -13,6 +13,8 @@ #define DMA_H2T_DESC_COUNT 4096 #define NUM_DMA_ENG_PER_DEVICE 132 // for v2 2 nc with each 16, +// Max H2D DMA engines per device +#define NUM_H2D_DMA_PER_DEVICE 8 #define NDMA_QUEUE_DUMMY_RING_DESC_COUNT 64 #define NDMA_QUEUE_DUMMY_RING_SIZE (NDMA_QUEUE_DUMMY_RING_DESC_COUNT * sizeof(union udma_desc)) @@ -42,6 +44,7 @@ struct ndma_h2d_dma_cmpltn_thread { struct task_struct *thread; wait_queue_head_t wait_queue; atomic64_t nonempty_ctxq_bitmap; + u32 rr_start_bit; volatile bool stop; }; @@ -80,80 +83,6 @@ struct ndma_h2d_compl_queue { int ndma_h2d_compl_queue_init(struct neuron_device *nd, struct ndma_h2d_compl_queue *compl_queue); void ndma_h2d_compl_queue_destroy(struct ndma_h2d_compl_queue *compl_queue); -/* - * H2D DMA Context Queue - * --------------------- - * A fixed-size circular buffer storing pointers to `ndma_h2t_zcdma_context`. - * The queue tracks DMA context lifecycles (submitted, pinned, or unpinned) - * and maintains four logical indices: - * - * 1. head - * - Index of the first *valid* (non-empty) entry in the queue. - * - The context at `head` may be in any ndma_zcdma_state except - * NDMA_COMPLETED. - * - * 2. tail - * - Index of the next free slot where a new context pointer will be - * inserted. - * - * 3. first_pinned_unsubmitted - * - Index of the earliest context that is pinned but not yet submitted. - * - If no such context exists, this is set to first_unpinned. - * - * 4. first_unpinned - * - Index of the earliest context that is unpinned. - * - If no such context exists, this is set to tail. - * - * Queue state conditions: - * - Empty: head == tail - * - Full: head == ((tail + 1) & (capacity - 1)) - * - Current size: (tail - head + capacity) & (capacity - 1) - * - * Example (capacity = 10): - * - * index: 0 1 2 3 4 5 6 7 8 9 - * entry: [ ] [S] [S] [S] [P] [P] [U] [U] [ ] [ ] - * ^ ^ ^ ^ - * | | | | - * H FPU FU T - * - * Legend: - * H = head - * T = tail - * FPU = first_pinned_unsubmitted - * FU = first_unpinned - * - * [C] = completed - * [S] = submitted - * [P] = pinned but unsubmitted - * [U] = unpinned - * [ ] = empty slot - * - * @entries: Array of DMA context entries in the queue - * @completion_pool: Pre-allocated pool of completion_ptr buffer - * @page_list_pool: Pre-allocated pool of page_list arrays - * @capacity_mask: Maximum number of entries the queue can hold and minus one, - * capacity = capacity_mask + 1 - * @head: Index pointing to the head of the queue - * @tail: Index pointing to the tail of the queue - * @first_pinned_unsubmitted: Index of the first pinned but not yet submitted entry - * @first_unpinned: Index of the first unpinned entry in the queue - * @nr_pinned_pages: Total count of pinned memory pages belonged to the queue - */ -struct ndma_ctx_queue { - struct ndma_h2t_zcdma_context *entries; - void *completion_pool; - void *page_list_pool; - u32 capacity_mask; - u32 head; - u32 tail; - u32 first_pinned_unsubmitted; - u32 first_unpinned; - u64 nr_pinned_pages; -}; - -int ndma_ctx_queue_init(struct ndma_ctx_queue *queue); -void ndma_ctx_queue_free(struct ndma_eng *eng, struct ndma_ring *ring, struct ndma_ctx_queue *queue); /* * dma context for both sync and async DMA operations @@ -174,18 +103,22 @@ struct ndma_h2t_dma_context { u64 offset; // initial offset for this transfer u64 remaining; // initial remaining for this transfer u64 outstanding; // outstanding data for this transfer - used to compute wait time. (vs pending) - int pending_transfers; // pending transfers for this context. Used to update ring ptrs + int pending_tx_transfers; // pending tx (m2s) transfers for this context. Used to update ring ptrs + int pending_rx_transfers; // pending rx (s2m) transfers for this context (may differ from tx) void *completion_ptr; // completion buffer pointer (host memory buffer we poll on for completions) }; +struct ndma_h2t_dma_ioq; + struct ndma_ring { // TODO combine all the h2t stuff in a sub structure struct mutex h2t_ring_lock; struct udma_ring_ptr h2t_completion; // TODO why are we using udma_ring_ptr... struct mem_chunk *h2t_completion_mc; struct ndma_h2t_dma_context h2t_dma_ctx[NEURON_DMA_H2T_CTX_HANDLE_CNT]; - struct ndma_ctx_queue dma_ctx_queue; + //TODO - these are blowing up ring size move them to h2t_dma_ioq struct ndma_h2d_compl_queue dma_compl_queue; + struct ndma_h2t_dma_ioq *h2t_dma_ioq; u32 h2t_nc_id; bool h2t_allocated; // ring can be allocated for standard use or h2t u32 qid; @@ -483,3 +416,4 @@ static inline bool ndmar_qid_valid(int qid) return ((qid >= 0) && (qid < DMA_MAX_Q_MAX)); } #endif + diff --git a/neuron_sysfs_metrics.c b/neuron_sysfs_metrics.c index 9212ef3..991d39b 100644 --- a/neuron_sysfs_metrics.c +++ b/neuron_sysfs_metrics.c @@ -13,10 +13,22 @@ #include "neuron_device.h" #include "neuron_ds.h" #include "neuron_fw_io.h" +#include "neuron_platform.h" #include "neuron_sysfs_metrics.h" #include "neuron_dhal.h" #include "neuron_power.h" +#include "neuron_ham_throttle.h" #include "share/neuron_driver_shared.h" +#include "neuron_cdev.h" + +#define CONNECTED_DEVICES_MAX_LEN 20 + +const struct attribute_group *neuron_class_groups[] = { + &neuron_soc_group, + &efa_group, + &platform_group, + NULL, +}; #define NEURON_ARCH_MAX_LEN 20 @@ -181,7 +193,7 @@ static const struct healthcheck_status_field healthcheck_status_fields[] = { { 9, 0x1, "link_efa" }, { 10, 0x1, "link_interserver" }, { 11, 0x1, "link_switch" }, - { 12, 0x1, "axi_fabric" }, + { 12, 0x1, "fabric_error" }, }; static const int healthcheck_status_fields_cnt = sizeof(healthcheck_status_fields) / sizeof(healthcheck_status_fields[0]); @@ -199,6 +211,14 @@ static const nsysfsmetric_attr_info_t root_arch_node_attrs_info_tbl[] = { }; static const int root_arch_node_attrs_info_tbl_cnt = sizeof(root_arch_node_attrs_info_tbl) / sizeof(nsysfsmetric_attr_info_t); +static const nsysfsmetric_attr_info_t root_info_node_attrs_info_tbl[] = { + ATTR_INFO("connected_devices", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_CONNECTED_DEVICES), OTHER), + ATTR_INFO("fw_api_version", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_FW_API_VERSION), OTHER), + ATTR_INFO("fw_build", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_FW_BUILD), OTHER), + ATTR_INFO("serial_number", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_SERIAL_NUMBER), OTHER), +}; +static const int root_info_node_attrs_info_tbl_cnt = sizeof(root_info_node_attrs_info_tbl) / sizeof(nsysfsmetric_attr_info_t); + static const nsysfsmetric_attr_info_t power_utilization_attrs_info_tbl[] = { ATTR_INFO("utilization", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_POWER_UTILIZATION), OTHER), ATTR_INFO("utilization_raw", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_POWER_UTILIZATION_RAW), OTHER), @@ -210,6 +230,17 @@ static const nsysfsmetric_attr_info_t tensor_engine_attrs_info_tbl[] = { }; static const int tensor_engine_attrs_info_tbl_cnt = sizeof(tensor_engine_attrs_info_tbl) / sizeof(nsysfsmetric_attr_info_t); +static const nsysfsmetric_attr_info_t ham_throttle_attrs_info_tbl[] = { + ATTR_INFO("throttle", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_HAM_THROTTLE), OTHER), +}; +static const int ham_throttle_attrs_info_tbl_cnt = sizeof(ham_throttle_attrs_info_tbl) / sizeof(nsysfsmetric_attr_info_t); + +static const nsysfsmetric_attr_info_t ctrl_node_attrs_info_tbl[] = { + ATTR_INFO("reset", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_RESET), OTHER), + ATTR_INFO("notify_delay", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_NOTIFY_DELAY), OTHER), +}; +static const int ctrl_node_attrs_info_tbl_cnt = sizeof(ctrl_node_attrs_info_tbl) / sizeof(nsysfsmetric_attr_info_t); + static void nsysfsmetric_node_release(struct kobject *kobj) { struct nsysfsmetric_node *node = container_of(kobj, struct nsysfsmetric_node, kobj); @@ -501,6 +532,54 @@ static ssize_t nsysfsmetric_show_nrt_other_metrics(struct nsysfsmetric_metrics * pr_err_ratelimited("sysfs failed to read pe_array activity counters, error = %d\n", ret); } len = nsysfsmetric_sysfs_emit(buf, "%s", buffer); + } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_HAM_THROTTLE)) { + struct neuron_device *nd = sysfs_metrics->nd; + + char buffer[256]; + neuron_ham_throttle_format(nd, attr->nc_id, buffer, sizeof(buffer)); + len = nsysfsmetric_sysfs_emit(buf, "%s", buffer); + } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_CONNECTED_DEVICES)) { + struct neuron_device *nd = sysfs_metrics->nd; + u32 connected_devices[MAX_NEURON_DEVICE_COUNT]; + int connected_device_count = 0; + int i, offset = 0; + int ret = ndhal->ndhal_fw_io.fw_io_topology(nd->fw_io_ctx, nd->pdev->device, nd->device_index, connected_devices, &connected_device_count); + if (ret) + return ret; + for (i = 0; i < connected_device_count - 1; i++) { + offset += snprintf(buf + offset, CONNECTED_DEVICES_MAX_LEN - offset, "%d, ", connected_devices[i]); + if (offset >= CONNECTED_DEVICES_MAX_LEN) { + pr_err("snprintf failed when showing connected devices\n"); + return 0; + } + } + if (connected_device_count == 0) + offset += snprintf(buf, CONNECTED_DEVICES_MAX_LEN, "\n"); + else + offset += snprintf(buf + offset, CONNECTED_DEVICES_MAX_LEN - offset, "%d\n", connected_devices[connected_device_count - 1]); + if (offset >= CONNECTED_DEVICES_MAX_LEN) { + pr_err("snprintf failed when showing connected devices\n"); + return 0; + } + len = offset; + } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_FW_API_VERSION)) { + struct neuron_device *nd = sysfs_metrics->nd; + u32 fw_api_version = 0; + fw_io_api_version_read(nd->npdev.bar0, &fw_api_version); + if (fw_api_version == 0xdeadbeef) // the value is not readable during reset, try later + len = nsysfsmetric_sysfs_emit(buf, "busy\n"); + else + len = nsysfsmetric_sysfs_emit(buf, "%u\n", fw_api_version); + } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_FW_BUILD)) { + struct neuron_device *nd = sysfs_metrics->nd; + u32 fw_build = 0; + int ret = fw_io_fw_build_read(nd->npdev.bar0, &fw_build); + if (ret || fw_build == 0xdeadbeef) // the value is not readable during reset, try later + len = nsysfsmetric_sysfs_emit(buf, "busy\n"); + else + len = nsysfsmetric_sysfs_emit(buf, "%u\n", fw_build); + } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_RESET)) { + len = nsysfsmetric_sysfs_emit(buf, "%d\n", sysfs_metrics->nd->device_state); } else { pr_err("cannot show sysfs metrics for nc_id=%d, metric_id=%d of attr_type OTHER \n", attr->nc_id, attr->metric_id); } @@ -652,6 +731,10 @@ static ssize_t nsysfsmetric_set_nrt_other_metrics(struct nsysfsmetric_metrics *s } else if (new_notify_delay == 0) { nsysfsmetric_notify = true; } + } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_HAM_THROTTLE)) { + neuron_ham_throttle_set_aggregate_count(sysfs_metrics->nd, attr->nc_id, buf, size); + } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_RESET)) { + ncdev_trigger_sysfs_reset(sysfs_metrics->nd); } else { pr_err("cannot set sysfs metrics for nc_id=%d, metric_id=%d of attr_type OTHER\n", attr->nc_id, attr->metric_id); } @@ -848,8 +931,28 @@ static int nsysfsmetric_init_and_add_nc_default_nodes(struct neuron_device *nd, return -1; } - // add the neuron_core{0,1, ...}/stats/status node and its children - struct nsysfsmetric_node *status_node = nsysfsmetric_init_and_add_one_node(sysfs_metrics, stats_node, "status", false, nc_id, 0, NULL); + // add the neuron_core{0,1, ...}/stats/hardware node + struct nsysfsmetric_node *hardware_node = nsysfsmetric_init_and_add_one_node(sysfs_metrics, stats_node, "hardware", false, -1, 0, NULL); + if (!hardware_node) { + pr_err("failed to create the hardware node under %s stats\n", nc_name); + return -1; + } + + // add the neuron_core{0,1, ...}/stats/hardware/tensor_engine node + ret = ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_tensor_engine_node(sysfs_metrics, hardware_node, nc_id, tensor_engine_attrs_info_tbl_cnt, tensor_engine_attrs_info_tbl); + if (ret) { + return ret; + } + + // add the neuron_core{0,1, ...}/stats/application node + struct nsysfsmetric_node *application_node = nsysfsmetric_init_and_add_one_node(sysfs_metrics, stats_node, "application", false, -1, 0, NULL); + if (!application_node) { + pr_err("failed to create the application node under %s stats\n", nc_name); + return -1; + } + + // add the neuron_core{0,1, ...}/stats/application/status node and its children + struct nsysfsmetric_node *status_node = nsysfsmetric_init_and_add_one_node(sysfs_metrics, application_node, "status", false, nc_id, 0, NULL); if (!status_node) { pr_err("failed to create the status node under %s\n", nc_name); return -1; @@ -860,6 +963,18 @@ static int nsysfsmetric_init_and_add_nc_default_nodes(struct neuron_device *nd, return ret; } + // add the neuron_core{0,1, ...}/stats/application/other_info node and its children + struct nsysfsmetric_node *custom_node = nsysfsmetric_init_and_add_one_node(sysfs_metrics, application_node, "other_info", false, nc_id, 0, NULL); + if (!custom_node) { + pr_err("failed to create the other_info node under %s\n", nc_name); + return -1; + } + ret = nsysfsmetric_init_and_add_nodes(sysfs_metrics, custom_node, custom_counter_nodes_info_tbl_cnt, custom_counter_nodes_info_tbl, nc_id); + if (ret) { + pr_err("failed to add the other_info node's children under %s\n", nc_name); + return ret; + } + // add the neuron_core{0,1, ...}/stats/memory_usage node and its children struct nsysfsmetric_node *mem_node = nsysfsmetric_init_and_add_one_node(sysfs_metrics, stats_node, "memory_usage", false, nc_id, 0, NULL); if (!mem_node) { @@ -884,22 +999,14 @@ static int nsysfsmetric_init_and_add_nc_default_nodes(struct neuron_device *nd, return ret; } - // add the neuron_core{0,1, ...}/stats/other_info node and its children - struct nsysfsmetric_node *custom_node = nsysfsmetric_init_and_add_one_node(sysfs_metrics, stats_node, "other_info", false, nc_id, 0, NULL); - if (!custom_node) { - pr_err("failed to create the other_info node under %s\n", nc_name); - return -1; - } - ret = nsysfsmetric_init_and_add_nodes(sysfs_metrics, custom_node, custom_counter_nodes_info_tbl_cnt, custom_counter_nodes_info_tbl, nc_id); - if (ret) { - pr_err("failed to add the other_info node's children under %s\n", nc_name); - return ret; - } - - // add the neuron_core{0,1, ...}/stats/tensor_engine node - ret = ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_tensor_engine_node(sysfs_metrics, stats_node, nc_id, tensor_engine_attrs_info_tbl_cnt, tensor_engine_attrs_info_tbl); - if (ret) { - return ret; + // add the neuron_core{0,1, ...}/stats/ham_throttle node. + if (narch_get_arch() == NEURON_ARCH_V3 || + narch_get_arch() == NEURON_ARCH_V4) { + struct nsysfsmetric_node *ham_throttle_node = nsysfsmetric_init_and_add_one_node(sysfs_metrics, hardware_node, "ham_throttle", false, nc_id, ham_throttle_attrs_info_tbl_cnt, ham_throttle_attrs_info_tbl); + if (!ham_throttle_node) { + pr_err("failed to create the ham_throttle node under %s hardware\n", nc_name); + return -1; + } } // add the neuron_core{0,1, ...}/info node @@ -908,6 +1015,11 @@ static int nsysfsmetric_init_and_add_nc_default_nodes(struct neuron_device *nd, pr_err("failed to add info under sysfs metric root\n"); return -1; } + // add the neuron_core{0,1, ...}/info/efa_device symlink + int efa_symlink_ret = nplatform_sysfs_add_efa_symlink(nd, &info_node->kobj, nc_id); + if (efa_symlink_ret) { + pr_err("nd%d nc:%d: failed to link efa_device: %d\n", nd->device_index, nc_id, efa_symlink_ret); + } // add the neuron_core{0,1, ...}/info/architecture node and its attributes struct nsysfsmetric_node *arch_node = nsysfsmetric_init_and_add_one_node(sysfs_metrics, info_node, "architecture", false, nc_id, arch_info_attrs_info_tbl_cnt, arch_info_attrs_info_tbl); if (!arch_node) { @@ -1083,7 +1195,7 @@ int nsysfsmetric_register(struct neuron_device *nd, struct kobject *neuron_devic } // neuron{0, 1, ...}/info/ - struct nsysfsmetric_node *info_node = nsysfsmetric_init_and_add_one_node(metrics, &metrics->root, "info", false, -1, ndhal->ndhal_sysfs_metrics.root_info_node_attrs_info_tbl_cnt, ndhal->ndhal_sysfs_metrics.root_info_node_attrs_info_tbl); + struct nsysfsmetric_node *info_node = nsysfsmetric_init_and_add_one_node(metrics, &metrics->root, "info", false, -1, root_info_node_attrs_info_tbl_cnt, root_info_node_attrs_info_tbl); if (!info_node) { pr_err("failed to add info node under sysfs metric root\n"); return -1; @@ -1120,19 +1232,39 @@ int nsysfsmetric_register(struct neuron_device *nd, struct kobject *neuron_devic return ret; } // neuron{0, 1, ...}/stats/hardware/{sram_ecc_uncorrected, mem_ecc_uncorrected} - ret = ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_ecc_nodes(metrics, stats_node, ecc_attrs_info_tbl_cnt, ecc_attrs_info_tbl); - if (ret) { - return ret; + struct nsysfsmetric_node *hardware_node = + nsysfsmetric_init_and_add_one_node(metrics, stats_node, "hardware", false, -1, + ecc_attrs_info_tbl_cnt, ecc_attrs_info_tbl); + if (!hardware_node) { + pr_err("failed to add hardware node under stats\n"); + return -1; } - // neuron{0, 1, ...}/stats/hardware/health_status/ - if (ndhal->ndhal_sysfs_metrics.health_status_enabled && metrics->hardware_node) { - ret = nsysfsmetric_add_health_status_nodes(metrics, metrics->hardware_node); + // neuron{0, 1, ...}/stats/hardware/power/ + struct nsysfsmetric_node *power_node = + nsysfsmetric_init_and_add_one_node(metrics, hardware_node, "power", false, -1, + power_utilization_attrs_info_tbl_cnt, + power_utilization_attrs_info_tbl); + if (!power_node) { + pr_err("failed to add power utilization sysfs node under stats/hardware/power\n"); + return -1; + } + + // neuron{0, 1, ...}/health_status/ + if (ndhal->ndhal_sysfs_metrics.health_status_enabled) { + ret = nsysfsmetric_add_health_status_nodes(metrics, &metrics->root); if (ret) { return ret; } } + // neuron{0, 1, ...}/ctrl/{reset, notify_delay} + struct nsysfsmetric_node *ctrl_node = nsysfsmetric_init_and_add_one_node(metrics, &metrics->root, "ctrl", false, -1, ctrl_node_attrs_info_tbl_cnt, ctrl_node_attrs_info_tbl); + if (!ctrl_node) { + pr_err("failed to add ctrl node under sysfs metric root\n"); + return -1; + } + // neuron{0, 1, ...}/neuron_core{0, 1, ...}/ ret = nsysfsmetric_init_and_add_nc_default_nodes(nd, &metrics->root); if (ret) { @@ -1140,16 +1272,6 @@ int nsysfsmetric_register(struct neuron_device *nd, struct kobject *neuron_devic return ret; } - // neuron{0, 1, ...}/stats/power - struct nsysfsmetric_node *power_node = - nsysfsmetric_init_and_add_one_node(metrics, stats_node, "power", false, -1, - power_utilization_attrs_info_tbl_cnt, - power_utilization_attrs_info_tbl); - if (!power_node) { - pr_err("failed to add power utilization sysfs node under stats/power\n"); - return -1; - } - return 0; } @@ -1162,7 +1284,6 @@ static void nsysfsmetric_destroy_counters(struct nsysfsmetric_metrics *metrics) memset(metrics->nrt_metrics, 0, sizeof(metrics->nrt_metrics)); memset(metrics->nrt_nd_metrics, 0, sizeof(metrics->nrt_nd_metrics)); memset(metrics->dev_metrics, 0, sizeof(metrics->dev_metrics)); - metrics->hardware_node = NULL; metrics->health_status_node = NULL; } @@ -1200,7 +1321,7 @@ void nsysfsmetric_destroy(struct neuron_device *nd) /* * Reads a configured subset of misc RAM registers and updates the cache exposed via - * stats/hardware/health_status/. Bumps hw_error_event_count and issues sysfs_notify on any change. + * health_status/. Bumps hw_error_event_count and issues sysfs_notify on any change. * Invoked periodically from the metrics thread. */ void nsysfsmetric_health_status_tick(struct neuron_device *nd) @@ -1245,15 +1366,23 @@ void nsysfsmetric_health_status_tick(struct neuron_device *nd) } } -int nsysfsmetric_add_health_status_nodes(struct nsysfsmetric_metrics *metrics, struct nsysfsmetric_node *hardware_node) +int nsysfsmetric_add_health_status_nodes(struct nsysfsmetric_metrics *metrics, struct nsysfsmetric_node *parent_node) { - struct nsysfsmetric_node *node = nsysfsmetric_init_and_add_one_node(metrics, hardware_node, + struct nsysfsmetric_node *node = nsysfsmetric_init_and_add_one_node(metrics, parent_node, "health_status", false, -1, health_status_attrs_info_tbl_cnt, health_status_attrs_info_tbl); if (!node) { - pr_err("failed to add health_status node under stats/hardware\n"); + pr_err("failed to add health_status node\n"); return -1; } metrics->health_status_node = node; + + // health_status/ecc_counters/; also exposed under stats/hardware/ for backwards compatibility + struct nsysfsmetric_node *ecc_counters_node = nsysfsmetric_init_and_add_one_node(metrics, node, + "ecc_counters", false, -1, ecc_attrs_info_tbl_cnt, ecc_attrs_info_tbl); + if (!ecc_counters_node) { + pr_err("failed to add ecc_counters node under health_status\n"); + return -1; + } return 0; } @@ -1383,3 +1512,4 @@ void nsysfsmetric_inc_reset_req_count(struct neuron_device *nd, int nc_id) { nsysfsmetric_inc_counter(nd, NON_NDS_METRIC, NON_NDS_COUNTER_RESET_REQ_COUNT, nc_id, 1, true); } + diff --git a/neuron_sysfs_metrics.h b/neuron_sysfs_metrics.h index 97bd261..2bd70b7 100644 --- a/neuron_sysfs_metrics.h +++ b/neuron_sysfs_metrics.h @@ -81,6 +81,11 @@ enum nsysfsmetric_non_nds_ids { // The metrics needed by sysfs metrics but not s NON_NDS_HEALTH_STATUS_OVERALL_HEALTH, NON_NDS_HEALTH_STATUS_HEARTBEAT, NON_NDS_OTHER_POWER_UTILIZATION_RAW, + NON_NDS_OTHER_HAM_THROTTLE, + NON_NDS_OTHER_CONNECTED_DEVICES, + NON_NDS_OTHER_FW_API_VERSION, + NON_NDS_OTHER_FW_BUILD, + NON_NDS_OTHER_RESET, }; struct neuron_device; @@ -131,7 +136,6 @@ struct nsysfsmetric_metrics { // per neuron_device u32 cached_health_regs[HEALTH_STATUS_SLOT_COUNT]; u32 hw_error_event_count; bool health_check_regs_read_failed; - struct nsysfsmetric_node *hardware_node; // stats/hardware/; cached so health_status can attach under it struct nsysfsmetric_node *health_status_node; // target for sysfs_notify on hw_error_event }; @@ -148,10 +152,12 @@ typedef struct nsysfsmetric_counter_node_info { nsysfsmetric_attr_info_t attr_info_tbl[MAX_COUNTER_ATTR_TYPE_COUNT]; // present, total, and/or peak } nsysfsmetric_counter_node_info_t; +extern const struct attribute_group *neuron_class_groups[]; + /** * nsysfsmetric_register() - Perform various sysfs inits such as kobj init and attribute group creation per neuron device * - * @nd: The pointer to the device which is the parent of the kobj to be initialized + * @nd: The pointer to the device which is the parent of the kobj to be initialized * @nd_kobj: The pointer to the parent kobject of nd->sysfs_metrics.metric_kobj */ int nsysfsmetric_register(struct neuron_device *nd, struct kobject *nd_kobj); @@ -239,9 +245,10 @@ void nsysfsmetric_set_counter(struct neuron_device *nd, int metric_id_category, void nsysfsmetric_inc_reset_fail_count(struct neuron_device *nd); /** - * nsysfsmetric_add_health_status_nodes() - add stats/hardware/health_status/ subtree under the hardware node + * nsysfsmetric_add_health_status_nodes() - add the health_status/ subtree (including the + * canonical ecc_counters/ location) under the given parent node */ -int nsysfsmetric_add_health_status_nodes(struct nsysfsmetric_metrics *metrics, struct nsysfsmetric_node *hardware_node); +int nsysfsmetric_add_health_status_nodes(struct nsysfsmetric_metrics *metrics, struct nsysfsmetric_node *parent_node); /** * nsysfsmetric_health_status_tick() - perform a single health_status cache refresh; invoked from the metrics thread @@ -249,3 +256,4 @@ int nsysfsmetric_add_health_status_nodes(struct nsysfsmetric_metrics *metrics, s void nsysfsmetric_health_status_tick(struct neuron_device *nd); #endif + diff --git a/neuron_sysfs_platform.c b/neuron_sysfs_platform.c new file mode 100644 index 0000000..4fa7a4e --- /dev/null +++ b/neuron_sysfs_platform.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + */ + +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "neuron_arch.h" +#include "neuron_device.h" +#include "neuron_dhal.h" +#include "neuron_pci.h" +#include "neuron_platform.h" + +static const char *const family_names[] = { + [NEURON_PLATFORM_FAMILY_NONE] = "none", + [NEURON_PLATFORM_FAMILY_INF2] = "inf2", + [NEURON_PLATFORM_FAMILY_TRN1] = "trn1", + [NEURON_PLATFORM_FAMILY_TRN1N] = "trn1n", + [NEURON_PLATFORM_FAMILY_TRN2] = "trn2", + [NEURON_PLATFORM_FAMILY_TRN2_ULTRASERVER] = "trn2_ultraserver", + [NEURON_PLATFORM_FAMILY_TRN3_GEN1_ULTRASERVER] = "trn3_gen1_ultraserver", + [NEURON_PLATFORM_FAMILY_TRN3_GEN2_ULTRASERVER] = "trn3_gen2_ultraserver", +}; +static_assert(ARRAY_SIZE(family_names) == NEURON_PLATFORM_FAMILY_TRN3_GEN2_ULTRASERVER + 1, + "family_names out of sync with enum neuron_platform_family"); + +static const char *const intra_topology_names[] = { + [NEURON_INTRA_RACK_TOPOLOGY_NONE] = "none", + [NEURON_INTRA_RACK_TOPOLOGY_RING] = "ring", + [NEURON_INTRA_RACK_TOPOLOGY_TORUS] = "torus", + [NEURON_INTRA_RACK_TOPOLOGY_SWITCH] = "switch", +}; +static_assert(ARRAY_SIZE(intra_topology_names) == NEURON_INTRA_RACK_TOPOLOGY_SWITCH + 1, + "intra_topology_names out of sync with enum neuron_intra_rack_topology"); + +static const char *const inter_topology_names[] = { + [NEURON_INTER_RACK_TOPOLOGY_NONE] = "none", + [NEURON_INTER_RACK_TOPOLOGY_P2P] = "p2p", +}; +static_assert(ARRAY_SIZE(inter_topology_names) == NEURON_INTER_RACK_TOPOLOGY_P2P + 1, + "inter_topology_names out of sync with enum neuron_inter_rack_topology"); + +static const char *const switch_type_names[] = { + [NEURON_SWITCH_TYPE_NONE] = "none", + [NEURON_SWITCH_TYPE_V1] = "v1", + [NEURON_SWITCH_TYPE_V2] = "v2", +}; +static_assert(ARRAY_SIZE(switch_type_names) == NEURON_SWITCH_TYPE_V2 + 1, + "switch_type_names out of sync with enum neuron_switch_type"); + +static const char *lookup_name(const char *const *tbl, size_t tbl_sz, unsigned int idx) +{ + if (idx >= tbl_sz || !tbl[idx]) + return "none"; + return tbl[idx]; +} + +#if (!defined(RHEL_RELEASE_CODE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0))) || (defined(RHEL_RELEASE_CODE) && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(9, 5))) +#define NPLATFORM_CLASS_SHOW_ARGS const struct class *cls, const struct class_attribute *attr +#else +#define NPLATFORM_CLASS_SHOW_ARGS struct class *cls, struct class_attribute *attr +#endif + +static ssize_t soc_version_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + if (!ndhal) + return 0; + return dhal_sysfs_emit(buf, "%d\n", ndhal->ndhal_arch.arch); +} + +static ssize_t sub_version_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + unsigned int sub_version = NEURON_SOC_SUB_VERSION_A0; + + if (!ndhal) + return 0; + if (ndhal->ndhal_arch.arch == NEURON_ARCH_V4 && narch_get_revision() >= 0x02) + sub_version = NEURON_SOC_SUB_VERSION_B1; + return dhal_sysfs_emit(buf, "%u\n", sub_version); +} +static CLASS_ATTR_RO(sub_version); + +static ssize_t hbm_capacity_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + if (!ndhal) + return 0; + return dhal_sysfs_emit(buf, "%llu\n", + ndhal->ndhal_address_map.hbm_active_size >> 30); +} +static CLASS_ATTR_RO(hbm_capacity); + +/* + * neuron_soc/version and efa/version share the same attribute name in + * different groups; declared with __ATTR to keep unique C symbols. + */ +static struct class_attribute class_attr_soc_version = __ATTR(version, S_IRUGO, soc_version_show, NULL); + +static struct attribute *neuron_soc_attrs[] = { + &class_attr_soc_version.attr, + &class_attr_sub_version.attr, + &class_attr_hbm_capacity.attr, + NULL, +}; +const struct attribute_group neuron_soc_group = { + .name = "neuron_soc", + .attrs = neuron_soc_attrs, +}; + +static ssize_t efa_version_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + return dhal_sysfs_emit(buf, "%d\n", neuron_pci_get_efa_version()); +} + +static struct class_attribute class_attr_efa_version = __ATTR(version, S_IRUGO, efa_version_show, NULL); + +static struct attribute *efa_attrs[] = { + &class_attr_efa_version.attr, + NULL, +}; +const struct attribute_group efa_group = { + .name = "efa", + .attrs = efa_attrs, +}; + +static ssize_t family_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + enum neuron_platform_family fam; + + if (!ndhal || !ndhal->ndhal_arch.platform) + return 0; + fam = ndhal->ndhal_arch.platform->family; + return dhal_sysfs_emit(buf, "%s\n", + lookup_name(family_names, ARRAY_SIZE(family_names), fam)); +} +static CLASS_ATTR_RO(family); + +static ssize_t num_chips_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + if (!ndhal || !ndhal->ndhal_arch.platform) + return 0; + return dhal_sysfs_emit(buf, "%u\n", ndhal->ndhal_arch.platform->device_count); +} +static CLASS_ATTR_RO(num_chips); + +static ssize_t num_sleds_per_server_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + if (!ndhal || !ndhal->ndhal_arch.platform) + return 0; + return dhal_sysfs_emit(buf, "%u\n", + ndhal->ndhal_arch.platform->num_sleds_per_server); +} +static CLASS_ATTR_RO(num_sleds_per_server); + +static ssize_t num_chips_per_sled_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + if (!ndhal || !ndhal->ndhal_arch.platform) + return 0; + return dhal_sysfs_emit(buf, "%u\n", + ndhal->ndhal_arch.platform->num_chips_per_sled); +} +static CLASS_ATTR_RO(num_chips_per_sled); + +static ssize_t num_cores_per_chip_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + if (!ndhal) + return 0; + return dhal_sysfs_emit(buf, "%d\n", ndhal->ndhal_address_map.nc_per_device); +} +static CLASS_ATTR_RO(num_cores_per_chip); + +static ssize_t supported_lnc_degrees_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + unsigned long mask; + int bit; + ssize_t n = 0; + + if (!ndhal) + return 0; + mask = ndhal->ndhal_address_map.supported_lnc_degrees; + for_each_set_bit(bit, &mask, BITS_PER_LONG) + n += scnprintf(buf + n, PAGE_SIZE - n, "%s%d", n ? " " : "", bit); + n += scnprintf(buf + n, PAGE_SIZE - n, "\n"); + return n; +} +static CLASS_ATTR_RO(supported_lnc_degrees); + +static ssize_t intra_rack_topology_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + enum neuron_intra_rack_topology t; + + if (!ndhal || !ndhal->ndhal_arch.platform) + return 0; + t = ndhal->ndhal_arch.platform->interconnect.intra_rack.topology; + return dhal_sysfs_emit(buf, "%s\n", + lookup_name(intra_topology_names, + ARRAY_SIZE(intra_topology_names), t)); +} +static CLASS_ATTR_RO(intra_rack_topology); + +static ssize_t intra_rack_switch_type_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + enum neuron_switch_type s; + + if (!ndhal || !ndhal->ndhal_arch.platform) + return 0; + s = ndhal->ndhal_arch.platform->interconnect.intra_rack.switch_type; + return dhal_sysfs_emit(buf, "%s\n", + lookup_name(switch_type_names, + ARRAY_SIZE(switch_type_names), s)); +} +static CLASS_ATTR_RO(intra_rack_switch_type); + +static ssize_t inter_rack_topology_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + enum neuron_inter_rack_topology t; + + if (!ndhal || !ndhal->ndhal_arch.platform) + return 0; + t = ndhal->ndhal_arch.platform->interconnect.inter_rack.topology; + return dhal_sysfs_emit(buf, "%s\n", + lookup_name(inter_topology_names, + ARRAY_SIZE(inter_topology_names), t)); +} +static CLASS_ATTR_RO(inter_rack_topology); + +static ssize_t inter_rack_switch_type_show(NPLATFORM_CLASS_SHOW_ARGS, char *buf) +{ + enum neuron_switch_type s; + + if (!ndhal || !ndhal->ndhal_arch.platform) + return 0; + s = ndhal->ndhal_arch.platform->interconnect.inter_rack.switch_type; + return dhal_sysfs_emit(buf, "%s\n", + lookup_name(switch_type_names, + ARRAY_SIZE(switch_type_names), s)); +} +static CLASS_ATTR_RO(inter_rack_switch_type); + +static struct attribute *platform_attrs[] = { + &class_attr_family.attr, + &class_attr_num_chips.attr, + &class_attr_num_sleds_per_server.attr, + &class_attr_num_chips_per_sled.attr, + &class_attr_num_cores_per_chip.attr, + &class_attr_supported_lnc_degrees.attr, + &class_attr_intra_rack_topology.attr, + &class_attr_intra_rack_switch_type.attr, + &class_attr_inter_rack_topology.attr, + &class_attr_inter_rack_switch_type.attr, + NULL, +}; + +const struct attribute_group platform_group = { + .name = "platform", + .attrs = platform_attrs, +}; + +int nplatform_sysfs_add_efa_symlink(struct neuron_device *nd, struct kobject *nc_kobj, int nc_id) +{ + const struct neuron_efa_bdf_map *m; + const char *bdf; + struct pci_dev *pdev; + unsigned int dom, bus, slot, func; + int global_nc_id; + int ret; + + if (!ndhal || !ndhal->ndhal_arch.platform) { + return 0; + } + + m = &ndhal->ndhal_arch.platform->nc_efa_bdf_tbl; + if (m->count == 0) { + return 0; + } + + global_nc_id = nd->device_index * ndhal->ndhal_address_map.nc_per_device + nc_id; + if (global_nc_id >= m->count) { + return -ENOENT; + } + + bdf = m->bdfs[global_nc_id]; + if (!bdf) { + return -ENOENT; + } + + if (sscanf(bdf, "%x:%x:%x.%x", &dom, &bus, &slot, &func) != 4) { + return -EINVAL; + } + + pdev = pci_get_domain_bus_and_slot(dom, bus, PCI_DEVFN(slot, func)); + if (!pdev) { + return -ENODEV; + } + + ret = sysfs_create_link(nc_kobj, &pdev->dev.kobj, "efa_device"); + pci_dev_put(pdev); + return ret; +} + diff --git a/neuron_test.c b/neuron_test.c index 8ed9b5f..ca487bc 100644 --- a/neuron_test.c +++ b/neuron_test.c @@ -78,3 +78,4 @@ int ntest_trigger(enum neuron_test_trigger trigger, u64 trigger_data) } return 0; } + diff --git a/neuron_test.h b/neuron_test.h index 51bdb31..6091b3d 100644 --- a/neuron_test.h +++ b/neuron_test.h @@ -27,3 +27,4 @@ static inline int _ntest_trigger(enum neuron_test_trigger trigger, u64 trigger_d return ntest_trigger(trigger, trigger_data); } #endif + diff --git a/neuron_topsp.c b/neuron_topsp.c index 21cf55f..bd3df7e 100644 --- a/neuron_topsp.c +++ b/neuron_topsp.c @@ -66,3 +66,4 @@ void ts_nq_destroy_all(struct neuron_device *nd) ndhal->ndhal_topsp.ts_nq_destroy_one(nd, ts_id); } } + diff --git a/neuron_topsp.h b/neuron_topsp.h index 945eb26..112de12 100644 --- a/neuron_topsp.h +++ b/neuron_topsp.h @@ -58,3 +58,4 @@ void ts_nq_destroy_all(struct neuron_device *nd); #endif + diff --git a/neuron_trace.h b/neuron_trace.h index 83f2a3c..ad1e1b9 100644 --- a/neuron_trace.h +++ b/neuron_trace.h @@ -389,3 +389,4 @@ TRACE_EVENT(metrics_post, #define TRACE_INCLUDE_PATH . #define TRACE_INCLUDE_FILE neuron_trace #include + diff --git a/postinstall b/postinstall index 393ad80..e2f44e5 100755 --- a/postinstall +++ b/postinstall @@ -1,4 +1,4 @@ -#!/usr/bin/sh -e +#!/bin/sh -e rm -f "/etc/modules-load.d/neuron.conf" echo "neuron" | tee -a /etc/modules-load.d/neuron.conf diff --git a/postremove b/postremove index d988ee3..3425aa9 100755 --- a/postremove +++ b/postremove @@ -1,4 +1,4 @@ -#!/usr/bin/sh -e +#!/bin/sh -e NEURON_PRES=$(dkms status | grep neuron) if [ -z "${NEURON_PRES}" ]; then diff --git a/share/neuron_driver_shared.h b/share/neuron_driver_shared.h index cb6ba9d..408a725 100644 --- a/share/neuron_driver_shared.h +++ b/share/neuron_driver_shared.h @@ -25,6 +25,15 @@ enum neuron_driver_feature_flag { NEURON_DRIVER_FEATURE_ALLOC_WITH_PA = 1ull << 10, NEURON_DRIVER_FEATURE_ASYNC_RW = 1ull << 11, // async H2D/D2H read/write NEURON_DRIVER_FEATURE_ASYNC_COPY = 1ull << 12, // async D2D copy + NEURON_DRIVER_FEATURE_H2D_DMA_NQ = 1ull << 13, // notification queues on H2D DMA engines + NEURON_DRIVER_FEATURE_ASYNC_COMPL = 1ull << 14, // async completion support + NEURON_DRIVER_FEATURE_H2D_DESC_EXECUTE = 1ull << 15, // NEURON_IOCTL_H2D_DESC_EXECUTE (user-built H2D descs) + NEURON_DRIVER_FEATURE_DMABUF_V2 = 1ull << 16, // dma-buf bug fixes + NEURON_DRIVER_FEATURE_ASYNC_MC2MC = 1ull << 17, // async MC to MC support plus Read After Write (RAW) support + NEURON_DRIVER_FEATURE_H2D_DMA_NQ_NC_ID = 1ull << 18, // notification init ioctls carry the owning physical NC in engine_index for H2D NQs + NEURON_DRIVER_FEATURE_ASYNC_IO_SUPPORTED = NEURON_DRIVER_FEATURE_ASYNC_MC2MC, // async IO fully supported. Requires, async RW, async completion, + // async desc execute, and async mc to mc (copy). MC2MC is last + // feature before full async IO support. Just alias that flag. }; enum neuron_switch_fabric_bar_type { @@ -35,7 +44,7 @@ enum neuron_switch_fabric_bar_type { }; // FIXME this should be more generic - like node type. -enum { +enum neuron_pod_type { NEURON_POD_TYPE_NONE = 0, NEURON_POD_TYPE_P2P, NEURON_POD_TYPE_SWITCH @@ -125,6 +134,7 @@ enum neuron_dma_h2t_ctx_handle_type { enum NQ_DEVICE_TYPE { NQ_DEVICE_TYPE_NEURON_CORE = 0, NQ_DEVICE_TYPE_TOPSP, + NQ_DEVICE_TYPE_H2D_DMA, /**< nq_dev_id is the H2D DMA engine id */ NQ_DEVICE_TYPE_MAX }; @@ -165,7 +175,7 @@ struct neuron_uuid { __u8 value[32]; }; -#define NEURON_MAX_PROCESS_PER_DEVICE 16 // 2 per core (arbitrary but needs to small number for fast lookup) +#define NEURON_MAX_PROCESS_PER_DEVICE 32 // 2 per core (arbitrary but needs to small number for fast lookup) #define APP_INFO_PID_NC_LOCK_INFO (1) #define APP_INFO_PID_MEM_USAGE (1 << 1) @@ -453,3 +463,4 @@ typedef struct nds_header { #define NDS_EXT_NEURONCORE_NC_DATA(base_addr, nc_index) ((uint64_t *)(base_addr + NDS_EXT_NEURONCORE_NC_DATA_START) + (nc_index * NDS_EXT_NEURONCORE_NC_DATA_COUNT)) #endif // NEURON_DRIVER_SHARED_H + diff --git a/share/neuron_driver_shared_tensor_batch_op.h b/share/neuron_driver_shared_tensor_batch_op.h index 59de68e..3c64f27 100644 --- a/share/neuron_driver_shared_tensor_batch_op.h +++ b/share/neuron_driver_shared_tensor_batch_op.h @@ -22,3 +22,4 @@ typedef struct nrt_tensor_batch_op { } nrt_tensor_batch_op_t; #endif // NEURON_DRIVER_SHARED_TENSOR_BATCH_OP_H + diff --git a/udma/udma.h b/udma/udma.h index 51400b7..ec41c1e 100644 --- a/udma/udma.h +++ b/udma/udma.h @@ -433,6 +433,33 @@ static inline u32 udma_ring_id_get(struct udma_q *udma_q) return ring_id; } +/** + * udma_ring_id_set() - Write the current ring id (phase bit) into a descriptor + * + * @udma_q: udma queue data structure the descriptor belongs to + * @desc: descriptor whose ring-id bits should be overwritten + * + * Advances the queue's phase-bit tracking (via udma_ring_id_get()) and + * rewrites the ring-id field of @desc's len_ctrl, using the M2S or S2M + * mask/shift as appropriate for the queue's direction. Any pre-existing + * ring-id bits in @desc are discarded. + * + * @note Like udma_ring_id_get(), this must be called exactly once per + * allocated descriptor to keep phase-bit tracking correct. + */ +static inline void udma_ring_id_set(struct udma_q *udma_q, union udma_desc *desc) +{ + u32 ring_id = udma_ring_id_get(udma_q); + + if (udma_q->type == UDMA_TX) { + desc->tx.len_ctrl = (desc->tx.len_ctrl & ~M2S_DESC_RING_ID_MASK) | + ((ring_id << M2S_DESC_RING_ID_SHIFT) & M2S_DESC_RING_ID_MASK); + } else { + desc->rx.len_ctrl = (desc->rx.len_ctrl & ~S2M_DESC_RING_ID_MASK) | + ((ring_id << S2M_DESC_RING_ID_SHIFT) & S2M_DESC_RING_ID_MASK); + } +} + /** * udma_desc_action_add() - Add number descriptors to the submission queue. * @@ -551,3 +578,4 @@ int udma_m2m_copy_prepare_one(struct udma *udma, u32 qid, dma_addr_t s_addr, dma int udma_m2m_copy_start(struct udma *udma, u32 qid, u32 m2s_count, u32 s2m_count); #endif + diff --git a/udma/udma_iofic.c b/udma/udma_iofic.c index 262e182..1d85eb8 100644 --- a/udma/udma_iofic.c +++ b/udma/udma_iofic.c @@ -134,6 +134,12 @@ /** Bus parity error on descriptor being prefetched */ #define INT_2ND_GROUP_B_S2M_PREFETCH_AXI_PARITY BIT(8) +/** Bus request to no descriptors hint - S2M engine has no RX descriptor available */ +#define INT_2ND_GROUP_B_S2M_NO_DESC_HINT BIT(7) + +/** Bus request to no descriptors timeout - S2M prefetcher timed out waiting for RX descriptor */ +#define INT_2ND_GROUP_B_S2M_NO_DESC_TO BIT(6) + /** Bus request to I/O Fabric timeout error */ #define INT_2ND_GROUP_B_S2M_DATA_AXI_TO BIT(2) /** Bus response from I/O Fabric error */ @@ -198,7 +204,7 @@ INT_2ND_GROUP_B_S2M_PREFETCH_RING_ID | INT_2ND_GROUP_B_S2M_PREFETCH_AXI_TO | \ INT_2ND_GROUP_B_S2M_PREFETCH_AXI_RESP | INT_2ND_GROUP_B_S2M_PREFETCH_AXI_PARITY | \ INT_2ND_GROUP_B_S2M_DATA_AXI_TO | INT_2ND_GROUP_B_S2M_DATA_AXI_RESP | \ - INT_2ND_GROUP_B_S2M_DATA_AXI_PARITY) + INT_2ND_GROUP_B_S2M_DATA_AXI_PARITY | INT_2ND_GROUP_B_S2M_NO_DESC_TO) #define UDMA_V4_IOFIC_2ND_GROUP_B_ERROR_INTS \ (UDMA_IOFIC_2ND_GROUP_B_ERROR_INTS | INT_2ND_GROUP_B_S2M_TRNSCTN_TBL_INFO_PARITY) @@ -376,6 +382,8 @@ void udma_iofic_s2m_error_ints_unmask(struct udma *udma) { u32 primary_grp_mask; u32 sec_b; + u32 err_log_mask; + u32 data_cfg_2; primary_grp_mask = INT_GROUP_D_S2M | INT_UDMA_V4_GROUP_D_2ND_IOFIC_GROUP_C; sec_b = UDMA_V4_IOFIC_2ND_GROUP_B_ERROR_INTS; @@ -401,6 +409,20 @@ void udma_iofic_s2m_error_ints_unmask(struct udma *udma) UDMA_V4_IOFIC_2ND_GROUP_C_S2M_ERROR_INTS); udma_iofic_unmask_adv(udma, UDMA_IOFIC_LEVEL_PRIMARY, INT_GROUP_D, primary_grp_mask); + + /* Mask rdr_no_desc_hint (bit 7) in S2M err_log_mask. The hint fires + * immediately when no descriptor is available and would initiate a log + * capture before the timeout (bit 6) fires. Masking the hint ensures the + * log only loads when the timeout triggers. + * Note: both bits still appear in the log data (the mask controls the load condition, not the data input to the flop). */ + reg_read32(&udma->udma_regs_s2m->s2m.err_log_mask, &err_log_mask); + err_log_mask |= INT_2ND_GROUP_B_S2M_NO_DESC_HINT; + reg_write32(&udma->udma_regs_s2m->s2m.err_log_mask, err_log_mask); + + /* Set no-desc timeout (desc_wait_timer) to maximum to avoid false aborts */ + reg_read32(&udma->udma_regs_s2m->s2m_wr.data_cfg_2, &data_cfg_2); + data_cfg_2 = (data_cfg_2 & 0xFF000000) | 0x00FFFFFF; + reg_write32(&udma->udma_regs_s2m->s2m_wr.data_cfg_2, data_cfg_2); } void udma_iofic_error_ints_unmask_one(struct iofic_grp_ctrl *iofic_ctrl, uint32_t mask) @@ -410,3 +432,4 @@ void udma_iofic_error_ints_unmask_one(struct iofic_grp_ctrl *iofic_ctrl, uint32_ iofic_unmask(iofic_ctrl, 0, mask); } + diff --git a/udma/udma_m2m.c b/udma/udma_m2m.c index c79fd9a..77fa198 100644 --- a/udma/udma_m2m.c +++ b/udma/udma_m2m.c @@ -527,3 +527,4 @@ void udma_m2m_mask_ring_id_error(struct udma *udma, void __iomem *intc_base) { reg_write32(&udma->udma_regs_s2m->s2m.err_log_mask, ring_id_mask); } + diff --git a/udma/udma_main.c b/udma/udma_main.c index 4147289..b2e7ac6 100644 --- a/udma/udma_main.c +++ b/udma/udma_main.c @@ -710,3 +710,4 @@ int udma_desc_action_add(struct udma_q *udma_q, u32 num) reg_write32(addr, num); return 0; } + diff --git a/udma/udma_regs.h b/udma/udma_regs.h index a24d589..080bbbb 100644 --- a/udma/udma_regs.h +++ b/udma/udma_regs.h @@ -333,6 +333,14 @@ struct udma_s2m_rd { u32 reserved0[12]; }; +struct udma_s2m_wr { + /* [0x0] Stream data FIFO configuration */ + u32 data_cfg_1; + /* [0x4] Data write configuration */ + u32 data_cfg_2; + u32 reserved0[14]; +}; + struct udma_s2m_comp { /* [0x0] Completion controller configuration */ u32 cfg_1c; @@ -390,7 +398,7 @@ struct udma_s2m_regs_v4 { struct udma_axi_s2m axi_s2m; /* [0x100] */ struct udma_s2m s2m; /* [0x200] */ struct udma_s2m_rd s2m_rd; /* [0x300] */ - u32 reserved1[16]; + struct udma_s2m_wr s2m_wr; /* [0x340] */ struct udma_s2m_comp s2m_comp; /* [0x380] */ u32 reserved2[784]; struct udma_s2m_q s2m_q[16]; /* [0x1000] */ @@ -666,3 +674,4 @@ union udma_q_regs { }; #endif + diff --git a/v2/address_map.h b/v2/address_map.h index 9c801f8..50798fd 100644 --- a/v2/address_map.h +++ b/v2/address_map.h @@ -197,3 +197,4 @@ #define V2_TPB_ARR_SEQ_QUEUE_PERF_IDLE_CYCLE_CNT_LSB_OFFSET 0x00000000000010ull #endif + diff --git a/v2/neuron_dhal_v2.c b/v2/neuron_dhal_v2.c index 592feb9..b97564d 100644 --- a/v2/neuron_dhal_v2.c +++ b/v2/neuron_dhal_v2.c @@ -458,6 +458,91 @@ static void nnq_set_hwaddr_v2(struct neuron_device *nd, u8 nc_id, u8 index, u32 notific_write_nq_f_size(apb_base, index, size); } +static const uint32_t v2_nnq_h2d_id_to_eng_id[] = {V2_D2H_IDX, V2_H2D_IDX}; + +static int nnq_get_nq_h2d_id_v2(uint32_t h2d_eng_id) +{ + int i; + for (i = 0; i < ARRAY_SIZE(v2_nnq_h2d_id_to_eng_id); i++) { + if (v2_nnq_h2d_id_to_eng_id[i] == h2d_eng_id) + return i; + } + return -1; +} + +static uint32_t nnq_get_h2d_id_v2(int nq_h2d_id) +{ + return v2_nnq_h2d_id_to_eng_id[nq_h2d_id]; +} + +static int nnq_set_h2d_hwaddr_v2(struct neuron_device *nd, uint32_t h2d_eng_id, u32 size, u64 queue_pa) +{ + u64 sdma_base; + void *apb_base; + u32 low, high; + + // H2D notification block lives in the same SDMA layout as per-NC SE engines, + // just hung off the H2D/D2H SDMA base instead of a SENG_x base. + if (h2d_eng_id == V2_H2D_IDX) + sdma_base = V2_APB_H2D_SDMA_BASE; + else if (h2d_eng_id == V2_D2H_IDX) + sdma_base = V2_APB_D2H_SDMA_BASE; + else + return -EINVAL; + + apb_base = nd->npdev.bar0 + V2_PCIE_BAR0_APB_OFFSET + (sdma_base - V2_APB_BASE) + + V2_APB_SENG_0_SDMA_0_NOTIFIC_RELBASE; + + low = (u32)(queue_pa & 0xffffffff); + high = (u32)(queue_pa >> 32U); + + notific_write_nq_base_addr_hi(apb_base, 0, high); + notific_write_nq_base_addr_lo(apb_base, 0, low); + notific_write_nq_f_size(apb_base, 0, size); + return 0; +} + +static int nnq_set_nq_h2d_enable_v2(struct neuron_device *nd, uint32_t h2d_eng_id, u32 enable) +{ + u64 sdma_base; + void *apb_base; + + // H2D notification block lives in the same SDMA layout as per-NC SE engines, + // just hung off the H2D/D2H SDMA base instead of a SENG_x base. + if (h2d_eng_id == V2_H2D_IDX) + sdma_base = V2_APB_H2D_SDMA_BASE; + else if (h2d_eng_id == V2_D2H_IDX) + sdma_base = V2_APB_D2H_SDMA_BASE; + else + return -EINVAL; + + apb_base = nd->npdev.bar0 + V2_PCIE_BAR0_APB_OFFSET + (sdma_base - V2_APB_BASE) + + V2_APB_SENG_0_SDMA_0_NOTIFIC_RELBASE; + + notific_write_nq_enable(apb_base, enable); + return 0; +} + +static int nnq_get_nq_h2d_enable_v2(struct neuron_device *nd, uint32_t h2d_eng_id, u32 *enable) +{ + u64 sdma_base; + void *apb_base; + + // H2D notification block lives in the same SDMA layout as per-NC SE engines, + // just hung off the H2D/D2H SDMA base instead of a SENG_x base. + if (h2d_eng_id == V2_H2D_IDX) + sdma_base = V2_APB_H2D_SDMA_BASE; + else if (h2d_eng_id == V2_D2H_IDX) + sdma_base = V2_APB_D2H_SDMA_BASE; + else + return -EINVAL; + + apb_base = nd->npdev.bar0 + V2_PCIE_BAR0_APB_OFFSET + (sdma_base - V2_APB_BASE) + + V2_APB_SENG_0_SDMA_0_NOTIFIC_RELBASE; + + *enable = notific_read_nq_enable(apb_base); + return 0; +} /* Memory Pool Functions */ /** @@ -533,8 +618,8 @@ static void ndmar_ctx_queue_from_bit_v2(int bit, uint32_t *h2d_eng_id, uint32_t *qid = 0; } -/** - * ndmar_is_h2t_def_q() - return true +/** + * ndmar_is_h2t_def_q() - return true * * @param nd: Neuron device which contains the DMA engine * @param eng_id: engine id @@ -739,11 +824,6 @@ static int fw_io_read_csr_array_v2(void **ptrs, u32 *values, u32 num_csrs, bool if (num_csrs > FW_IO_MAX_READLESS_READ_REGISTER_COUNT) return -EINVAL; - // Force virtual platforms onto the direct path - if (narch_is_qemu() || narch_is_emu()) { - fw_io_read_csr_array_direct(ptrs, values, num_csrs, operational); - } - return fw_io_read_csr_array_direct(ptrs, values, num_csrs, operational); } @@ -805,37 +885,6 @@ static int mmap_get_bar4_offset_v2(u64 start_addr, u64 size, u64 *offset) /* Sysfs Metrics Functions */ // sysfs root node's attrs and its child nodes' attrs -static nsysfsmetric_attr_info_t root_info_node_attrs_info_tbl_v2[] = { - ATTR_INFO("notify_delay", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_NOTIFY_DELAY), OTHER), - ATTR_INFO("serial_number", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_SERIAL_NUMBER), OTHER), -}; -static int root_info_node_attrs_info_tbl_cnt_v2 = sizeof(root_info_node_attrs_info_tbl_v2) / sizeof(nsysfsmetric_attr_info_t); - -/** - * nsysfsmetric_add_ecc_nodes() - add neuron{0, 1, ...}/stats/hardware/{sram_ecc_uncorrected, mem_ecc_uncorrected} to sysfs directory - * - * @param metrics: the sysfs metrics structure - * @param stats_node: the sysfs node structure of the stats directory - * @param ecc_attrs_info_tbl_cnt: number of the ecc attributes - * @param attr_info_tbl: the ecc attributes as an array - * @return int 0 on success; otherwise on failure - * - * Note: ecc errors are only supported by sysfs for V2. - */ -static int nsysfsmetric_add_ecc_nodes_v2(struct nsysfsmetric_metrics *metrics, - struct nsysfsmetric_node *stats_node, - int ecc_attrs_info_tbl_cnt, - const nsysfsmetric_attr_info_t *ecc_attrs_info_tbl) -{ - struct nsysfsmetric_node *hardware_node = nsysfsmetric_init_and_add_one_node(metrics, stats_node, "hardware", false, -1, ecc_attrs_info_tbl_cnt, ecc_attrs_info_tbl); - if (!hardware_node) { - pr_err("failed to add hardware node its attributes under stats\n"); - return -1; - } - - return 0; -} - /** * nsysfsmetric_get_hbm_error_count_v2() - check hbm repairable/unrepairable error count * @@ -881,6 +930,172 @@ static int nsysfsmetric_add_tensor_engine_node_v2(struct nsysfsmetric_metrics *m /* PCI Functions */ +static const struct neuron_platform_desc v2_platform_std = { + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_NONE, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 16, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .nc_efa_bdf_tbl = { .bdfs = trn1_nc_efa_bdf_tbl, .count = sizeof(trn1_nc_efa_bdf_tbl) / sizeof(trn1_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 2, + .num_chips_per_sled = 8, +}; + +static const char *const v2_trn1_2xl_instance_names[] = { + "trn1.2xlarge", + NULL, +}; + +static const char *const v2_trn1_instance_names[] = { + "trn1.32xlarge", + NULL, +}; + +static const char *const v2_trn1n_instance_names[] = { + "trn1n.32xlarge", + NULL, +}; + +static const char *const v2_inf2_small_instance_names[] = { + "inf2.xlarge", + "inf2.8xlarge", + NULL, +}; + +static const char *const v2_inf2_24xl_instance_names[] = { + "inf2.24xlarge", + NULL, +}; + +static const char *const v2_inf2_48xl_instance_names[] = { + "inf2.48xlarge", + NULL, +}; + +static const char *const v2_inf2e_instance_names[] = { + "inf2e.32xlarge", + NULL, +}; + +static const struct neuron_platform_desc v2_platform_table[] = { + { + .instance_names = v2_trn1_2xl_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_TRN1, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 1, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .num_sleds_per_server = 1, + .num_chips_per_sled = 1, + }, + { + .instance_names = v2_trn1_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_TRN1, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 16, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_TORUS } }, + .nc_efa_bdf_tbl = { .bdfs = trn1_nc_efa_bdf_tbl, .count = sizeof(trn1_nc_efa_bdf_tbl) / sizeof(trn1_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 2, + .num_chips_per_sled = 8, + }, + { + .instance_names = v2_trn1n_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_TRN1N, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 16, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_TORUS } }, + .nc_efa_bdf_tbl = { .bdfs = trn1n_nc_efa_bdf_tbl, .count = sizeof(trn1n_nc_efa_bdf_tbl) / sizeof(trn1n_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 2, + .num_chips_per_sled = 8, + }, + { + .instance_names = v2_inf2_24xl_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_INF2, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 6, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_RING } }, + .num_sleds_per_server = 1, + .num_chips_per_sled = 6, + }, + { + .instance_names = v2_inf2_small_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_INF2, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 1, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .num_sleds_per_server = 1, + .num_chips_per_sled = 1, + }, + { + .instance_names = v2_inf2_48xl_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_INF2, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 12, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_RING } }, + .num_sleds_per_server = 1, + .num_chips_per_sled = 12, + }, + { + .instance_names = v2_inf2e_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_TRN1N, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 16, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_TORUS } }, + .nc_efa_bdf_tbl = { .bdfs = trn1n_nc_efa_bdf_tbl, .count = sizeof(trn1n_nc_efa_bdf_tbl) / sizeof(trn1n_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 2, + .num_chips_per_sled = 8, + }, + { } +}; + +static const struct neuron_platform_desc *ndhal_select_platform_v2(void) +{ + const struct neuron_platform_desc *platform; + + platform = nplatform_get_platform_desc(v2_platform_table); + + return platform ? platform : &v2_platform_std; +} + +static int neuron_pci_init_v2(struct neuron_device *nd) { + return 0; +} + // for V2 rename Neuron devices for better customer experience. // see internal documentation: TRN1-Discovery // map routing id to user id: @@ -1037,6 +1252,9 @@ static void ndma_get_wait_for_completion_time_v2(u32 count, bool async, u64 *fir // https://tiny.amazon.com/8jw7wl18 // In the meantime make the timeout 100x the original *following_wait_time *= 100; + if (narch_is_qemu() || narch_is_emu()) { + *following_wait_time *= 100 * 1000; + } } /** @@ -1423,6 +1641,7 @@ int ndhal_register_funcs_v2(void) { return -EINVAL; } + ndhal->ndhal_arch.platform = ndhal_select_platform_v2(); ndhal->ndhal_arch.narch_platform_ready = narch_platform_ready_v2; ndhal->ndhal_address_map.pci_host_base = V2_PCIE_A0_BASE; ndhal->ndhal_address_map.mmap_nc_event_offset = V2_MMAP_NC_EVENT_OFFSET; @@ -1439,10 +1658,14 @@ int ndhal_register_funcs_v2(void) { ndhal->ndhal_address_map.ts_per_device = V2_TS_PER_DEVICE; ndhal->ndhal_address_map.dma_eng_per_nc = V2_DMA_ENG_PER_NC; ndhal->ndhal_address_map.num_hbms = V2_NUM_HBMS; + ndhal->ndhal_address_map.hbm_active_size = V2_HBM_0_SIZE; ndhal->ndhal_address_map.nq_per_nc = V2_MAX_NQ_SUPPORTED; + ndhal->ndhal_address_map.supported_lnc_degrees = BIT(1); ndhal->ndhal_reset.initiate_max_wait_time = V2_NR_RESET_INIT_MAX_TOTAL_WAIT_TIME_MS; ndhal->ndhal_reset.retry_count = NR_RESET_RETRY_COUNT; ndhal->ndhal_reset.nr_post_reset_config = nr_post_reset_config_v2; + ndhal->ndhal_fw_metrics.max_misc_ram_version = 0; // v2 never negotiates the fw HBM layout + ndhal->ndhal_fw_metrics.parse_payload = NULL; ndhal->ndhal_topsp.ts_nq_init = ts_nq_init_v2; ndhal->ndhal_topsp.ts_nq_destroy_one = ts_nq_destroy_one_v2; ndhal->ndhal_topsp.ts_nq_get_nqid = ts_nq_get_nqid_v2; @@ -1452,6 +1675,12 @@ int ndhal_register_funcs_v2(void) { ndhal->ndhal_nq.nnq_get_nqid = nnq_get_nqid_v2; ndhal->ndhal_nq.nnq_set_hwaddr = nnq_set_hwaddr_v2; ndhal->ndhal_nq.nnq_get_nq_queue_count = nnq_get_nq_queue_count_v2; + ndhal->ndhal_nq.nnq_get_nq_h2d_id = nnq_get_nq_h2d_id_v2; + ndhal->ndhal_nq.nnq_get_h2d_id = nnq_get_h2d_id_v2; + ndhal->ndhal_nq.h2d_dma_nq_nc_id_supported = true; + ndhal->ndhal_nq.nnq_set_h2d_hwaddr = nnq_set_h2d_hwaddr_v2; + ndhal->ndhal_nq.nnq_set_nq_h2d_enable = nnq_set_nq_h2d_enable_v2; + ndhal->ndhal_nq.nnq_get_nq_h2d_enable = nnq_get_nq_h2d_enable_v2; ndhal->ndhal_mpset.mp_min_alloc_size = (mempool_min_alloc_size < 1024) ? 1024 : mempool_min_alloc_size; // v2 has a bigger mem size and gen pool create fails if < 1024 ndhal->ndhal_mpset.small_pool_supported = true; ndhal->ndhal_mpset.mpset_set_dram_and_mpset_info = mpset_set_dram_and_mpset_info_v2; @@ -1471,14 +1700,12 @@ int ndhal_register_funcs_v2(void) { ndhal->ndhal_fw_io.fw_io_post_metric = fw_io_post_metric_v2; ndhal->ndhal_mmap.dm_mmap_special = dm_mmap_special_v2; ndhal->ndhal_mmap.mmap_get_bar4_offset = mmap_get_bar4_offset_v2; - ndhal->ndhal_sysfs_metrics.root_info_node_attrs_info_tbl_cnt = root_info_node_attrs_info_tbl_cnt_v2; - ndhal->ndhal_sysfs_metrics.root_info_node_attrs_info_tbl = root_info_node_attrs_info_tbl_v2; - ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_ecc_nodes = nsysfsmetric_add_ecc_nodes_v2; ndhal->ndhal_sysfs_metrics.nsysfsmetric_get_hbm_error_count = nsysfsmetric_get_hbm_error_count_v2; ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_tensor_engine_node = nsysfsmetric_add_tensor_engine_node_v2; ndhal->ndhal_sysfs_metrics.health_status_enabled = false; ndhal->ndhal_pci.axi_bar = BAR_UNUSED; ndhal->ndhal_pci.dram_bar = 4; + ndhal->ndhal_pci.neuron_pci_init = neuron_pci_init_v2; ndhal->ndhal_pci.neuron_pci_get_device_id = neuron_pci_get_device_id_v2; ndhal->ndhal_pci.neuron_pci_device_id_to_rid_map = neuron_pci_device_id_to_rid_map_v2; ndhal->ndhal_cdev.ncdev_mem_regions = ncdev_mem_regions_v2; @@ -1518,12 +1745,28 @@ int ndhal_register_funcs_v2(void) { ndhal->ndhal_ext_cleanup = ndhal_ext_cleanup_v2; if (narch_is_qemu()) { + // initialize qemu setups to full device by default + if (nc_per_dev_param == 0) { + nc_per_dev_param = V2_NC_PER_DEVICE; + } + if (dev_nc_map == 0) { + dev_nc_map = ((1 << V2_NC_PER_DEVICE) - 1); + } ndhal->ndhal_reset.nr_initiate_reset = nr_initiate_reset_v2_qemu; ndhal->ndhal_reset.nr_wait_for_reset_completion = nr_wait_for_reset_completion_v2_qemu; - ndhal->ndhal_address_map.seng_dma_eng_per_nd = V2_NC_PER_DEVICE * V2_DMA_ENG_PER_NC; - ndhal->ndhal_address_map.h2d_dma_eng_per_nd = V2_NUM_H2D_DMA_PER_DEVICE; + ndhal->ndhal_address_map.seng_dma_eng_per_nd = nc_per_dev_param * V2_DMA_ENG_PER_NC; + ndhal->ndhal_address_map.h2d_dma_eng_per_nd = nc_per_dev_param; + ndhal->ndhal_address_map.nc_per_device = nc_per_dev_param; + ndhal->ndhal_address_map.dev_nc_map = dev_nc_map; ndhal->ndhal_pci.apb_bar = 2; } else if (narch_is_emu()) { + // initialize emu setups to single-tpb by default + if (nc_per_dev_param == 0) { + nc_per_dev_param = 1; + } + if (dev_nc_map == 0) { + dev_nc_map = 1; + } ndhal->ndhal_reset.retry_count *= 1000; // wait longer on the emulator ndhal->ndhal_reset.nr_initiate_reset = nr_initiate_reset_v2_emu; ndhal->ndhal_reset.nr_wait_for_reset_completion = nr_wait_for_reset_completion_v2_emu; @@ -1567,3 +1810,4 @@ int ndhal_register_funcs_v2(void) { return ret; } + diff --git a/v2/notific.c b/v2/notific.c index ed5ea6d..e49d0d9 100644 --- a/v2/notific.c +++ b/v2/notific.c @@ -123,3 +123,4 @@ int notific_decode_nq_head_reg_access_v2(u64 offset, u8 *nc_id, u32 *nq_type, u8 return -EINVAL; } + diff --git a/v2/notific.h b/v2/notific.h index f25ccf8..2f5624a 100644 --- a/v2/notific.h +++ b/v2/notific.h @@ -50,3 +50,4 @@ static inline u64 notific_get_relative_offset_topsp_v2(int ts_idx) int notific_decode_nq_head_reg_access_v2(u64 offset, u8 *nc_id, u32 *nq_type, u8 *instance, bool *is_top_sp); + diff --git a/v2/sdma.h b/v2/sdma.h index 72db929..fda7a2d 100644 --- a/v2/sdma.h +++ b/v2/sdma.h @@ -103,3 +103,4 @@ static inline int sdma_configure_broadcast(void __iomem *regs_base, int eng_id) return 0; } + diff --git a/v3/address_map.h b/v3/address_map.h index ad073c1..7799c8c 100644 --- a/v3/address_map.h +++ b/v3/address_map.h @@ -227,3 +227,4 @@ #define V3_TPB_ARR_SEQ_QUEUE_PERF_IDLE_CYCLE_CNT_LSB_OFFSET 0x00000000000010ull #endif + diff --git a/v3/neuron_dhal_v3.c b/v3/neuron_dhal_v3.c index 172c7ed..b734439 100644 --- a/v3/neuron_dhal_v3.c +++ b/v3/neuron_dhal_v3.c @@ -16,6 +16,7 @@ #include "../neuron_core.h" #include "../neuron_dma.h" #include "../neuron_fw_io.h" +#include "../neuron_fw_layout.h" #include "../neuron_pci.h" #include "../neuron_trace.h" #include "../neuron_cdev.h" @@ -23,6 +24,7 @@ #include "../neuron_ring.h" #include "../neuron_mempool.h" #include "neuron_pelect.h" +#include "neuron_fw_metrics.h" extern int dev_nc_map; extern int reset_top_dma; @@ -229,30 +231,6 @@ static int ndhal_register_funcs_trn2(void) { return 0; } -static const struct neuron_platform_lookup v3_platform_map[] = { - {"trn2p.48xlarge", NEURON_PLATFORM_TYPE_ULTRASERVER}, - {"trn2eu.48xlarge", NEURON_PLATFORM_TYPE_ULTRASERVER}, - {"trn2u.48xlarge", NEURON_PLATFORM_TYPE_ULTRASERVER}, - {"trn2u-ac.24xlarge", NEURON_PLATFORM_TYPE_ULTRASERVER}, - {"trn2es.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn2en.24xlarge", NEURON_PLATFORM_TYPE_MAX}, - {NULL, NEURON_PLATFORM_TYPE_INVALID}, -}; - -static enum neuron_platform_type ndhal_platform_type_v3(void) -{ - enum neuron_platform_type platform_type; - - platform_type = narch_detect_platform_type(v3_platform_map); - - if (force_userver) { - platform_type = NEURON_PLATFORM_TYPE_ULTRASERVER; - } - - return platform_type; -} - - /* Device Arch Functions */ /** * narch_platform_ready() - return platform ready status @@ -261,7 +239,7 @@ static enum neuron_platform_type ndhal_platform_type_v3(void) */ static int narch_platform_ready_v3(struct neuron_device *nd, enum neuron_platform_operation_type platform_operation) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_STD) { + if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_STD) { return 0; } else { return npe_platform_ready(nd, platform_operation); @@ -433,7 +411,12 @@ static int nr_post_reset_config_v3(struct neuron_device *nd, bool reset_successf } nd->current_perf_profile = 0; - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_STD) { + // firmware is back up (or, in no-reset mode, was never down): have the + // HBM layout reader negotiate on its next poll + if (reset_successful) + fw_layout_request_renegotiate(nd); + + if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_STD) { return 0; } @@ -614,6 +597,114 @@ static u8 nnq_get_nq_queue_count_v3(void) return V3_MAX_NQ_QUEUES; } +static int nnq_get_h2d_nq_base_v3(uint32_t h2d_eng_id, u64 *udma_base, u64 *io_base, u64 *bar0_io_offset) +{ + + // Top-level H2D/D2H DMA engines look like per-SE SDMA blocks at a + // different base, so reuse the SDMA notification register layout but + // anchor it off the matching H2D/D2H UDMA base. + switch (h2d_eng_id) { + case V3_D2H_0_IDX: + *udma_base = V3_APB_IO_0_D2H_UDMA_BASE; + *io_base = V3_APB_IO_0_BASE; + *bar0_io_offset = V3_PCIE_BAR0_APB_IO_0_OFFSET; + break; + case V3_H2D_0_IDX: + *udma_base = V3_APB_IO_0_H2D_UDMA_BASE; + *io_base = V3_APB_IO_0_BASE; + *bar0_io_offset = V3_PCIE_BAR0_APB_IO_0_OFFSET; + break; + case V3_D2H_1_IDX: + *udma_base = V3_APB_IO_1_D2H_UDMA_BASE; + *io_base = V3_APB_IO_1_BASE; + *bar0_io_offset = V3_PCIE_BAR0_APB_IO_1_OFFSET; + break; + case V3_H2D_1_IDX: + *udma_base = V3_APB_IO_1_H2D_UDMA_BASE; + *io_base = V3_APB_IO_1_BASE; + *bar0_io_offset = V3_PCIE_BAR0_APB_IO_1_OFFSET; + break; + default: + return -EINVAL; + } + + return 0; +} + +static const uint32_t v3_nnq_h2d_id_to_eng_id[] = { + V3_D2H_0_IDX, V3_H2D_0_IDX, V3_D2H_1_IDX, V3_H2D_1_IDX +}; + +static int nnq_get_nq_h2d_id_v3(uint32_t h2d_eng_id) +{ + int i; + for (i = 0; i < ARRAY_SIZE(v3_nnq_h2d_id_to_eng_id); i++) { + if (v3_nnq_h2d_id_to_eng_id[i] == h2d_eng_id) + return i; + } + return -1; +} + +static uint32_t nnq_get_h2d_id_v3(int nq_h2d_id) +{ + return v3_nnq_h2d_id_to_eng_id[nq_h2d_id]; +} + +static int nnq_set_h2d_hwaddr_v3(struct neuron_device *nd, uint32_t h2d_eng_id, + u32 size, u64 queue_pa) +{ + u64 udma_base, io_base, bar0_io_offset; + void *apb_base; + u32 low, high; + int ret = nnq_get_h2d_nq_base_v3(h2d_eng_id, &udma_base, &io_base, &bar0_io_offset); + if (ret != 0) { + return ret; + } + + apb_base = nd->npdev.bar0 + (udma_base - io_base) + bar0_io_offset + + V3_APB_SDMA_MISC_OFFSET + V3_APB_SENG_0_SDMA_0_NOTIFIC_RELBASE; + + low = (u32)(queue_pa & 0xffffffff); + high = (u32)(queue_pa >> 32U); + + notific_write_nq_base_addr_hi(apb_base, 0, high); + notific_write_nq_base_addr_lo(apb_base, 0, low); + notific_write_nq_f_size(apb_base, 0, size); + return 0; +} + +static int nnq_set_nq_h2d_enable_v3(struct neuron_device *nd, uint32_t h2d_eng_id, u32 enable) +{ + u64 udma_base, io_base, bar0_io_offset; + void *apb_base; + int ret = nnq_get_h2d_nq_base_v3(h2d_eng_id, &udma_base, &io_base, &bar0_io_offset); + if (ret != 0) { + return ret; + } + + apb_base = nd->npdev.bar0 + (udma_base - io_base) + bar0_io_offset + + V3_APB_SDMA_MISC_OFFSET + V3_APB_SENG_0_SDMA_0_NOTIFIC_RELBASE; + + notific_write_nq_enable(apb_base, enable); + return 0; +} + +static int nnq_get_nq_h2d_enable_v3(struct neuron_device *nd, uint32_t h2d_eng_id, u32 *enable) +{ + u64 udma_base, io_base, bar0_io_offset; + void *apb_base; + int ret = nnq_get_h2d_nq_base_v3(h2d_eng_id, &udma_base, &io_base, &bar0_io_offset); + if (ret != 0) { + return -EINVAL; + } + + apb_base = nd->npdev.bar0 + (udma_base - io_base) + bar0_io_offset + + V3_APB_SDMA_MISC_OFFSET + V3_APB_SENG_0_SDMA_0_NOTIFIC_RELBASE; + + *enable = notific_read_nq_enable(apb_base); + return 0; +} + /** * nnq_set_hwaddr() - set the physical address of the queue * @@ -952,11 +1043,6 @@ static int fw_io_read_csr_array_v3(void **ptrs, u32 *values, u32 num_csrs, bool if (num_csrs > FW_IO_MAX_READLESS_READ_REGISTER_COUNT) return -EINVAL; - // Force virtual platforms onto the direct path - if (narch_is_qemu() || narch_is_emu()) { - fw_io_read_csr_array_direct(ptrs, values, num_csrs, operational); - } - return fw_io_read_csr_array_direct(ptrs, values, num_csrs, operational); } @@ -1023,38 +1109,6 @@ static int mmap_get_bar4_offset_v3(u64 start_addr, u64 size, u64 *offset) /* Sysfs Metrics Functions */ // sysfs root node's attrs and its child nodes' attrs -static nsysfsmetric_attr_info_t root_info_node_attrs_info_tbl_v3[] = { - ATTR_INFO("notify_delay", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_NOTIFY_DELAY), OTHER), - ATTR_INFO("serial_number", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_SERIAL_NUMBER), OTHER), -}; -static int root_info_node_attrs_info_tbl_cnt_v3 = sizeof(root_info_node_attrs_info_tbl_v3) / sizeof(nsysfsmetric_attr_info_t); - -/** - * nsysfsmetric_add_ecc_nodes() - add neuron{0, 1, ...}/stats/hardware/{sram_ecc_uncorrected, mem_ecc_uncorrected} to sysfs directory - * - * @param metrics: the sysfs metrics structure - * @param stats_node: the sysfs node structure of the stats directory - * @param ecc_attrs_info_tbl_cnt: number of the ecc attributes - * @param attr_info_tbl: the ecc attributes as an array - * @return int 0 on success; otherwise on failure - * - * Note: ecc errors are only supported by sysfs for V2/3. - */ -static int nsysfsmetric_add_ecc_nodes_v3(struct nsysfsmetric_metrics *metrics, - struct nsysfsmetric_node *stats_node, - int ecc_attrs_info_tbl_cnt, - const nsysfsmetric_attr_info_t *ecc_attrs_info_tbl) -{ - struct nsysfsmetric_node *hardware_node = nsysfsmetric_init_and_add_one_node(metrics, stats_node, "hardware", false, -1, ecc_attrs_info_tbl_cnt, ecc_attrs_info_tbl); - if (!hardware_node) { - pr_err("failed to add hardware node its attributes under stats\n"); - return -1; - } - metrics->hardware_node = hardware_node; - - return 0; -} - /** * nsysfsmetric_get_hbm_error_count_v3() - check hbm repairable/unrepairable error count * @@ -1142,35 +1196,174 @@ static const u32 v3_torus_routing_id_to_user_id[] = { 1, 2, 5, 6, 13, 14, 9, 10 }; -// map routing id to user id for trn2pds instance type. -// the only hard rule this map needs to follow is -// rid (i*2) and rid (i*2)+1 map to did (n*2) and did (n*2)+1 -// since rid (i*2) and rid (i*2)+1 are on the same JBOG. -static const u32 v3_pds_routing_id_to_user_id[] = { - 0, 1, - 2, 3, - 4, 5, - 6, 7, - 8, 9, - 10, 11, - 12, 13, - 14, 15 }; - -static const u32 v3_max_routing_id_to_user_id[] = { - 0, 1, 2, 3 }; - #define V3_ROUTING_ID_TBL_SZ (sizeof(v3_torus_routing_id_to_user_id) / sizeof(v3_torus_routing_id_to_user_id[0])) -#define V3_MAX_ROUTING_ID_TBL_SZ (sizeof(v3_max_routing_id_to_user_id) / sizeof(v3_max_routing_id_to_user_id[0])) -static u32 neuron_pci_routing_id_to_user_id(u32 routing_id) +static const struct neuron_platform_desc v3_platform_std = { + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_TRN2, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_TABLE, .table = { v3_torus_routing_id_to_user_id, V3_ROUTING_ID_TBL_SZ } }, + .device_count = 16, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_TORUS } }, + .nc_efa_bdf_tbl = { .bdfs = trn2_nc_efa_bdf_tbl, .count = sizeof(trn2_nc_efa_bdf_tbl) / sizeof(trn2_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 8, + .num_chips_per_sled = 2, +}; + +static const char *const v3_trn2_instance_names[] = { + "trn2.48xlarge", + NULL, +}; + +static const char *const v3_trn2n_instance_names[] = { + "trn2n.48xlarge", + NULL, +}; + +static const char *const v3_ultraserver_instance_names[] = { + "trn2p.48xlarge", + "trn2eu.48xlarge", + "trn2u.48xlarge", + "trn2u-ac.24xlarge", + NULL, +}; + +static const char *const v3_pds_instance_names[] = { + "trn2es.48xlarge", + NULL, +}; + +static const char *const v3_max_instance_names[] = { + "trn2en.24xlarge", + NULL, +}; + +static const char *const v3_3xl_instance_names[] = { + "trn2.3xlarge", + NULL, +}; + +static const struct neuron_platform_desc v3_platform_table[] = { + { + .instance_names = v3_trn2_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_TRN2, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_TABLE, .table = { v3_torus_routing_id_to_user_id, V3_ROUTING_ID_TBL_SZ } }, + .device_count = 16, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_TORUS } }, + .nc_efa_bdf_tbl = { .bdfs = trn2_nc_efa_bdf_tbl, .count = sizeof(trn2_nc_efa_bdf_tbl) / sizeof(trn2_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 8, + .num_chips_per_sled = 2, + }, + { + .instance_names = v3_trn2n_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_TRN2, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_TABLE, .table = { v3_torus_routing_id_to_user_id, V3_ROUTING_ID_TBL_SZ } }, + .device_count = 16, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_TORUS } }, + .nc_efa_bdf_tbl = { .bdfs = trn2n_nc_efa_bdf_tbl, .count = sizeof(trn2n_nc_efa_bdf_tbl) / sizeof(trn2n_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 8, + .num_chips_per_sled = 2, + }, + { + .instance_names = v3_ultraserver_instance_names, + .type = NEURON_PLATFORM_TYPE_ULTRASERVER, + .family = NEURON_PLATFORM_FAMILY_TRN2_ULTRASERVER, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_TABLE, .table = { v3_torus_routing_id_to_user_id, V3_ROUTING_ID_TBL_SZ } }, + .device_count = 16, + .default_node_cnt = 4, + .pod_type = NEURON_POD_TYPE_P2P, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_ELECTION, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_TORUS }, .inter_rack = { .topology = NEURON_INTER_RACK_TOPOLOGY_P2P } }, + .nc_efa_bdf_tbl = { .bdfs = trn2_nc_efa_bdf_tbl, .count = sizeof(trn2_nc_efa_bdf_tbl) / sizeof(trn2_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 8, + .num_chips_per_sled = 2, + }, + { + .instance_names = v3_pds_instance_names, + .type = NEURON_PLATFORM_TYPE_PDS, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 16, + .default_node_cnt = 4, + .pod_type = NEURON_POD_TYPE_SWITCH, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_FW_IO, + .max_bars = 1, + .num_sleds_per_server = 8, + .num_chips_per_sled = 2, + }, + { + .instance_names = v3_max_instance_names, + .type = NEURON_PLATFORM_TYPE_MAX, + .family = NEURON_PLATFORM_FAMILY_TRN3_GEN2_ULTRASERVER, /* trn2max is a dev platform on the deprecation path; use TRN3_GEN2_ULTRASERVER as it is the publicly documented family for MAX topology */ + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 4, + .default_node_cnt = 1, + .pod_type = NEURON_POD_TYPE_SWITCH, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_FW_IO, + .max_bars = 2, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_SWITCH, .switch_type = NEURON_SWITCH_TYPE_V1 }, .inter_rack = { .topology = NEURON_INTER_RACK_TOPOLOGY_P2P } }, + .num_sleds_per_server = 1, + .num_chips_per_sled = 4, + }, + { + .instance_names = v3_3xl_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_TRN2, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 1, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .num_sleds_per_server = 1, + .num_chips_per_sled = 1, + }, + { } +}; + +static const struct neuron_platform_desc *ndhal_select_platform_v3(void) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS) { - return v3_pds_routing_id_to_user_id[ routing_id % V3_ROUTING_ID_TBL_SZ]; + const struct neuron_platform_desc *platform; + int i; + + if (force_userver) { + for (i = 0; v3_platform_table[i].instance_names != NULL; i++) { + if (v3_platform_table[i].type == NEURON_PLATFORM_TYPE_ULTRASERVER) { + return &v3_platform_table[i]; + } + } } - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { - return v3_max_routing_id_to_user_id[ routing_id % V3_MAX_ROUTING_ID_TBL_SZ]; + + platform = nplatform_get_platform_desc(v3_platform_table); + + return platform ? platform : &v3_platform_std; +} + +static int neuron_pci_init_v3(struct neuron_device *nd) { + return 0; +} + +static u32 neuron_pci_routing_id_to_user_id(u32 routing_id) +{ + u32 index = routing_id % ndhal->ndhal_arch.platform->device_count; + + if (ndhal->ndhal_arch.platform->routing_config.type == NEURON_PLATFORM_ROUTING_TYPE_TABLE) { + return ndhal->ndhal_arch.platform->routing_config.table.map[index]; } - return v3_torus_routing_id_to_user_id[ routing_id % V3_ROUTING_ID_TBL_SZ]; + return index; } static atomic_t v3_trn2max_device_count = ATOMIC_INIT(0); @@ -1191,7 +1384,7 @@ static int neuron_pci_get_device_id_v3(struct neuron_device *nd, struct pci_dev // FW does not populate routing_id miscram regs on Trn2MAX servers, // so assign device IDs sequentially by probe order instead. - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_MAX) { routing_id = (u32)dhal_atomic_fetch_add(1, &v3_trn2max_device_count); nd->device_index = neuron_pci_routing_id_to_user_id(routing_id); goto done; @@ -1211,9 +1404,10 @@ static int neuron_pci_get_device_id_v3(struct neuron_device *nd, struct pci_dev return -ENODEV; } - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS) { + // TODO: replace platform->type check with inter-rack topology type + if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_PDS) { int server_id; - + ret = fw_io_server_info_read(nd->npdev.bar0, &server_id, NULL); if (ret) { return -ENODEV; @@ -1262,7 +1456,8 @@ neuron_pci_device_id_to_rid_map_v3(uint32_t * count, uint32_t * did_to_rid_map) for (i = 0; i < total_neuron_devices; i++) { u32 routing_id; - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS) { + // TODO: replace platform->type check with inter-rack topology type + if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_PDS) { routing_id = i + ndhal->ndhal_arch.server_id * total_neuron_devices; } else { routing_id = i; @@ -1344,6 +1539,9 @@ static void ndma_get_wait_for_completion_time_v3(u32 count, bool async, u64 *fir // https://tiny.amazon.com/8jw7wl18 // In the meantime make the timeout 100x the original *following_wait_time *= 100; + if (narch_is_qemu() || narch_is_emu()) { + *following_wait_time *= 100 * 1000; + } } /** @@ -1553,7 +1751,7 @@ static int ndma_is_bar0_write_blocked_v3(u64 off) static int ndma_get_m2m_barrier_type_v3(bool set_dmb) { if (set_dmb) - return UDMA_M2M_BARRIER_WRITE_BARRIER; + return UDMA_M2M_BARRIER_DMB; else return UDMA_M2M_BARRIER_NONE; } @@ -1597,7 +1795,7 @@ static bool ndhal_die_flipped(enum neuron_ioctl_nc_mapping_type version) if (force_die_flip) { return true; } - if (!(ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_ULTRASERVER)) { + if (ndhal->ndhal_arch.platform->type != NEURON_PLATFORM_TYPE_ULTRASERVER) { return false; } @@ -1657,6 +1855,18 @@ static void ndma_get_engines_with_host_connectivity_v3(u32 hbm_index, u32 engine /* POD Functions */ +/** + * npe_platform_has_pod() - does the platform discover pod membership? + * + * A platform that runs no pod discovery never populates pod state, so the pod + * surfaces below answer from the platform descriptor instead of reporting state + * that stays perpetually undiscovered. + */ +static bool npe_platform_has_pod(void) +{ + return ndhal->ndhal_arch.platform->pod_discovery != NEURON_POD_DISCOVERY_TYPE_NONE; +} + /** * npe_notify_mark() - api for crwl to notify range marking (core claiming) activities * @@ -1666,7 +1876,7 @@ static void ndma_get_engines_with_host_connectivity_v3(u32 hbm_index, u32 engine */ static void npe_notify_mark_v3(int mark_cnt, bool mark) { - if (ndhal->ndhal_arch.platform_type != NEURON_PLATFORM_TYPE_STD) { + if (npe_platform_has_pod()) { npe_notify_mark(mark_cnt, mark); } } @@ -1683,19 +1893,20 @@ static void npe_notify_mark_v3(int mark_cnt, bool mark) */ static int npe_pod_info_v3(u8 *pod_type, u8 *pod_id, u8 *pod_sz, enum neuron_ultraserver_mode *mode, u32 *modes_supported) { - if ((ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_STD)) { + if (!npe_platform_has_pod()) { *pod_type = NEURON_POD_TYPE_NONE; *pod_sz = 0; *mode = NEURON_ULTRASERVER_MODE_UNSET; *modes_supported = 0; - } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + // TODO: replace platform->type check with inter-rack topology type + } else if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_PDS || + ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_MAX) { *pod_type = NEURON_POD_TYPE_SWITCH; npe_get_pod_sz(pod_sz); npe_get_pod_id(pod_id); npe_get_pod_mode(mode); npe_get_pod_modes_supported(modes_supported); - } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_ULTRASERVER) { + } else if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_ULTRASERVER) { *pod_type = NEURON_POD_TYPE_P2P; npe_get_pod_sz(pod_sz); npe_get_pod_id(pod_id); @@ -1717,7 +1928,7 @@ static int npe_pod_info_v3(u8 *pod_type, u8 *pod_id, u8 *pod_sz, enum neuron_ult */ static int npe_pod_status_v3(u32 *pod_state, s8 *node_id) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_STD) { + if (!npe_platform_has_pod()) { *pod_state = NEURON_POD_E_STATE_SINGLE_NODE; *node_id = -1; return 0; @@ -1737,7 +1948,7 @@ static int npe_pod_status_v3(u32 *pod_state, s8 *node_id) */ static int npe_pod_ctrl_v3(struct neuron_device *nd, u32 pod_ctrl, enum neuron_ultraserver_mode mode, u32 timeout, u32 *pod_state) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_STD) { + if (!npe_platform_has_pod()) { return 0; } return npe_pod_ctrl(nd, pod_ctrl, mode, timeout, pod_state); @@ -1746,6 +1957,10 @@ static int npe_pod_ctrl_v3(struct neuron_device *nd, u32 pod_ctrl, enum neuron_u static int perf_set_profile_v3(struct neuron_device *nd, uint32_t profile) { int ret; + int init_ret; + + ndmar_close_ncs(nd, NEURON_NC_MAP_DEVICE); + ret = fw_io_set_power_profile(nd->fw_io_ctx, profile); if (ret == 0) { ndhal->ndhal_perf.current_performance_profile = profile; @@ -1760,7 +1975,20 @@ static int perf_set_profile_v3(struct neuron_device *nd, uint32_t profile) nd->current_perf_profile = -1; } } - return ret; + + /* + * fw_io_set_power_profile triggers a PIR. + * Re-init DMAs after so `udma_set_defaults` runs on all engines including + * the top DMAs (which TPB resets don't init unless they're coalesced per seng). + * Without this, `spare_reg.zeroes0` bit0 stays at its HW default and + * top DMA completion writebacks never reach host memory. + */ + init_ret = ndmar_init_ncs(nd, NEURON_NC_MAP_DEVICE); + if (init_ret) { + pr_err("nd%d: failed to initialize dma after perf profile set\n", nd->device_index); + } + + return ret ? ret : init_ret; } static int perf_get_profile_v3(struct neuron_device *nd, uint32_t *profile) @@ -1804,13 +2032,7 @@ static void perf_update_hbm_7200_supported_v3(struct neuron_device *nd) static void fw_io_cache_bar_info_v3(struct neuron_device *nd) { int bar_idx; - int max_bars = 0; - - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { - max_bars = 2; - } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS) { - max_bars = 1; - } + int max_bars = ndhal->ndhal_arch.platform->max_bars; for (bar_idx = 0; bar_idx < max_bars; bar_idx++) { if (nd->bar_info[bar_idx].cached) { @@ -1832,7 +2054,7 @@ static void fw_io_cache_bar_info_v3(struct neuron_device *nd) */ static ssize_t npe_class_node_id_show_data_v3(char *buf, u32 sz) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_STD) { + if (!npe_platform_has_pod()) { return dhal_sysfs_emit(buf, "-1\n"); } return npe_class_node_id_show_data(buf, sz); @@ -1846,8 +2068,7 @@ static ssize_t npe_class_node_id_show_data_v3(char *buf, u32 sz) */ static ssize_t npe_class_node_cnt_show_data_v3(char *buf) { - if (ndhal->ndhal_arch.platform_type != NEURON_PLATFORM_TYPE_PDS && - ndhal->ndhal_arch.platform_type != NEURON_PLATFORM_TYPE_MAX) { + if (ndhal->ndhal_arch.platform->pod_discovery != NEURON_POD_DISCOVERY_TYPE_FW_IO) { return dhal_sysfs_emit(buf, "-1\n"); } return npe_class_node_cnt_show_data(buf); @@ -1862,7 +2083,7 @@ static ssize_t npe_class_node_cnt_show_data_v3(char *buf) */ static ssize_t npe_class_server_id_show_data_v3(char *buf, u32 sz) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_STD) { + if (!npe_platform_has_pod()) { return dhal_sysfs_emit(buf, "0000000000000000\n"); } return npe_class_server_id_show_data(buf, sz); @@ -1876,7 +2097,7 @@ static ssize_t npe_class_server_id_show_data_v3(char *buf, u32 sz) */ static ssize_t npe_class_ultraserver_mode_show_data_v3(char *buf) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_STD) { + if (!npe_platform_has_pod()) { return dhal_sysfs_emit(buf, "\n"); } return npe_class_ultraserver_mode_show_data(buf); @@ -1918,7 +2139,7 @@ static int ntpb_pe_get_aggregated_wl_cycle_cnt_v3(struct neuron_device *nd, int */ static void ndhal_ext_cleanup_v3(void) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_ULTRASERVER) { + if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_ELECTION) { npe_cleanup(); } return; @@ -1948,7 +2169,7 @@ int ndhal_register_funcs_v3(void) { return -EINVAL; } - ndhal->ndhal_arch.platform_type = ndhal_platform_type_v3(); + ndhal->ndhal_arch.platform = ndhal_select_platform_v3(); ndhal->ndhal_arch.narch_platform_ready = narch_platform_ready_v3; ndhal->ndhal_address_map.pci_host_base = V3_PCIE_A0_BASE; ndhal->ndhal_address_map.mmap_nc_event_offset = V3_MMAP_NC_EVENT_OFFSET; @@ -1965,10 +2186,13 @@ int ndhal_register_funcs_v3(void) { ndhal->ndhal_address_map.ts_per_device = V3_TS_PER_DEVICE; ndhal->ndhal_address_map.dma_eng_per_nc = V3_DMA_ENG_PER_NC; ndhal->ndhal_address_map.num_hbms = V3_NUM_HBMS; + ndhal->ndhal_address_map.hbm_active_size = V3_HBM_ACTIVE_SIZE; ndhal->ndhal_address_map.nq_per_nc = V3_MAX_NQ_SUPPORTED; + ndhal->ndhal_address_map.supported_lnc_degrees = BIT(1) | BIT(2); ndhal->ndhal_reset.initiate_max_wait_time = V3_NR_RESET_INIT_MAX_TOTAL_WAIT_TIME_MS; ndhal->ndhal_reset.retry_count = NR_RESET_RETRY_COUNT; ndhal->ndhal_reset.nr_post_reset_config = nr_post_reset_config_v3; + fw_metrics_init_reader_v3(); ndhal->ndhal_topsp.ts_nq_init = ts_nq_init_v3; ndhal->ndhal_topsp.ts_nq_destroy_one = ts_nq_destroy_one_v3; ndhal->ndhal_topsp.ts_nq_get_nqid = ts_nq_get_nqid_v3; @@ -1978,6 +2202,12 @@ int ndhal_register_funcs_v3(void) { ndhal->ndhal_nq.nnq_get_nqid = nnq_get_nqid_v3; ndhal->ndhal_nq.nnq_set_hwaddr = nnq_set_hwaddr_v3; ndhal->ndhal_nq.nnq_get_nq_queue_count = nnq_get_nq_queue_count_v3; + ndhal->ndhal_nq.nnq_get_nq_h2d_id = nnq_get_nq_h2d_id_v3; + ndhal->ndhal_nq.nnq_get_h2d_id = nnq_get_h2d_id_v3; + ndhal->ndhal_nq.h2d_dma_nq_nc_id_supported = true; + ndhal->ndhal_nq.nnq_set_h2d_hwaddr = nnq_set_h2d_hwaddr_v3; + ndhal->ndhal_nq.nnq_set_nq_h2d_enable = nnq_set_nq_h2d_enable_v3; + ndhal->ndhal_nq.nnq_get_nq_h2d_enable = nnq_get_nq_h2d_enable_v3; ndhal->ndhal_mpset.mp_min_alloc_size = (mempool_min_alloc_size < 1024) ? 1024 : mempool_min_alloc_size; ndhal->ndhal_mpset.small_pool_supported = true; ndhal->ndhal_mpset.mpset_set_dram_and_mpset_info = mpset_set_dram_and_mpset_info_v3; @@ -1998,15 +2228,13 @@ int ndhal_register_funcs_v3(void) { ndhal->ndhal_fw_io.fw_io_cache_bar_info = fw_io_cache_bar_info_v3; ndhal->ndhal_mmap.dm_mmap_special = dm_mmap_special_v3; ndhal->ndhal_mmap.mmap_get_bar4_offset = mmap_get_bar4_offset_v3; - ndhal->ndhal_sysfs_metrics.root_info_node_attrs_info_tbl_cnt = root_info_node_attrs_info_tbl_cnt_v3; - ndhal->ndhal_sysfs_metrics.root_info_node_attrs_info_tbl = root_info_node_attrs_info_tbl_v3; - ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_ecc_nodes = nsysfsmetric_add_ecc_nodes_v3; ndhal->ndhal_sysfs_metrics.nsysfsmetric_get_hbm_error_count = nsysfsmetric_get_hbm_error_count_v3; ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_tensor_engine_node = nsysfsmetric_add_tensor_engine_node_v3; ndhal->ndhal_sysfs_metrics.health_status_enabled = true; ndhal->ndhal_pci.axi_bar = BAR_UNUSED; ndhal->ndhal_pci.apb_bar = 0; ndhal->ndhal_pci.dram_bar = 4; + ndhal->ndhal_pci.neuron_pci_init = neuron_pci_init_v3; ndhal->ndhal_pci.neuron_pci_get_device_id = neuron_pci_get_device_id_v3; ndhal->ndhal_pci.neuron_pci_device_id_to_rid_map = neuron_pci_device_id_to_rid_map_v3; ndhal->ndhal_cdev.ncdev_mem_regions = ncdev_mem_regions_v3; @@ -2050,21 +2278,37 @@ int ndhal_register_funcs_v3(void) { extern unsigned int nmetric_log_posts; if (narch_is_qemu()) { + // initialize qemu setups to full device by default + if (nc_per_dev_param == 0) { + nc_per_dev_param = V3_NC_PER_DEVICE; + } + if (dev_nc_map == 0) { + dev_nc_map = ((1 << V3_NC_PER_DEVICE) - 1); + } ndhal->ndhal_reset.retry_count *= 1000; // wait longer on qemu ndhal->ndhal_reset.nr_initiate_reset = nr_initiate_reset_v3_qemu; ndhal->ndhal_reset.nr_wait_for_reset_completion = nr_wait_for_reset_completion_v3_qemu; - ndhal->ndhal_address_map.seng_dma_eng_per_nd = V3_NC_PER_DEVICE * V3_DMA_ENG_PER_NC; - ndhal->ndhal_address_map.h2d_dma_eng_per_nd = V3_NUM_H2D_DMA_PER_DEVICE; + ndhal->ndhal_address_map.seng_dma_eng_per_nd = nc_per_dev_param * V3_DMA_ENG_PER_NC; + ndhal->ndhal_address_map.h2d_dma_eng_per_nd = (nc_per_dev_param + V3_NC_PER_SENG - 1) / V3_NC_PER_SENG; + ndhal->ndhal_address_map.nc_per_device = nc_per_dev_param; + ndhal->ndhal_address_map.dev_nc_map = dev_nc_map; ndhal->ndhal_address_map.dice_per_device = 1; // Disable metrics on qemu nmetric_log_posts = 0; } else if (narch_is_emu()) { + // initialize emu setups to single-tpb by default + if (nc_per_dev_param == 0) { + nc_per_dev_param = 1; + } + if (dev_nc_map == 0) { + dev_nc_map = 1; + } ndhal->ndhal_reset.retry_count *= 1000; // wait longer on the emulator ndhal->ndhal_reset.nr_initiate_reset = nr_initiate_reset_v3_emu; ndhal->ndhal_reset.nr_wait_for_reset_completion = nr_wait_for_reset_completion_v3_emu; ndhal->ndhal_address_map.seng_dma_eng_per_nd = nc_per_dev_param * V3_DMA_ENG_PER_NC; - ndhal->ndhal_address_map.h2d_dma_eng_per_nd = nc_per_dev_param; + ndhal->ndhal_address_map.h2d_dma_eng_per_nd = (nc_per_dev_param + V3_NC_PER_SENG - 1) / V3_NC_PER_SENG; ndhal->ndhal_address_map.nc_per_device = nc_per_dev_param; ndhal->ndhal_address_map.dev_nc_map = dev_nc_map; ndhal->ndhal_address_map.dice_per_device = 1; @@ -2078,15 +2322,13 @@ int ndhal_register_funcs_v3(void) { ndhal->ndhal_address_map.h2d_dma_eng_per_nd = V3_NUM_H2D_DMA_PER_DEVICE; } - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_ULTRASERVER) { + if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_ELECTION) { ret = npe_init(); if (ret) { pr_err("failed to initialize pod election on V3\n"); return ret; } - } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS) { - // TODO PDS - } + } switch (ndhal->pci_device_id) { case TRN2_DEVICE_ID0: @@ -2112,3 +2354,4 @@ int ndhal_register_funcs_v3(void) { return ret; } + diff --git a/v3/neuron_fw_metrics.c b/v3/neuron_fw_metrics.c new file mode 100644 index 0000000..ca0fc26 --- /dev/null +++ b/v3/neuron_fw_metrics.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + * + * v3-specific firmware metrics: the supported misc-ram version and the payload + * parsers (shared helpers in ../neuron_fw_metrics_common.h). + */ + +#include + +#include "../neuron_dhal.h" +#include "neuron_fw_metrics.h" + +// no payload schema is adopted on v3 yet +static int fw_metrics_parse_payload_v3(struct neuron_device *nd, u16 type_id, const u8 *payload, u32 length) +{ + return -ENOTSUPP; +} + +void fw_metrics_init_reader_v3(void) +{ + ndhal->ndhal_fw_metrics.max_misc_ram_version = V3_FW_METRICS_MAX_MISC_RAM_VERSION; + ndhal->ndhal_fw_metrics.parse_payload = fw_metrics_parse_payload_v3; +} + diff --git a/v3/neuron_fw_metrics.h b/v3/neuron_fw_metrics.h new file mode 100644 index 0000000..e0f6987 --- /dev/null +++ b/v3/neuron_fw_metrics.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + * + * v3-specific firmware metrics; shared code lives in ../neuron_fw_metrics_common.h. + */ + +#ifndef NEURON_FW_METRICS_V3_H +#define NEURON_FW_METRICS_V3_H + +// highest misc-ram version whose metrics this driver parses on v3 (sent via SET_MAX_VERSION) +#define V3_FW_METRICS_MAX_MISC_RAM_VERSION 0x00 // none adopted yet + +void fw_metrics_init_reader_v3(void); + +#endif + diff --git a/v3/neuron_pelect.c b/v3/neuron_pelect.c index 84c971f..704dc8e 100644 --- a/v3/neuron_pelect.c +++ b/v3/neuron_pelect.c @@ -1130,7 +1130,7 @@ static void npe_initiate_election(u64 nbr_data_read_timeout) static bool npe_all_rst_complete(void) { int i; - int device_count = (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) ? 4 : 16; + int device_count = ndhal->ndhal_arch.platform->device_count; for (i = 0; i < device_count; i++) { if (ndhal_pelect_data.pnd[i] == NULL) { return false; @@ -1170,7 +1170,7 @@ int npe_election_exec_on_rst(struct neuron_device *nd, bool reset_successful) // Device 0 is the primary actor in the election/topology discovery process, so // when we process Device 0 reset completions, we need to do some bookkeeping. // - if ((nd->device_index == 0) && (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_ULTRASERVER)) { + if ((nd->device_index == 0) && (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_ELECTION)) { // Prior election results are cached in miscram, for testing purposes, // we can clear the results through a module parameter, allowing us // to ignore the cached results. @@ -1212,8 +1212,7 @@ int npe_election_exec_on_rst(struct neuron_device *nd, bool reset_successful) // ctl. // if ((ndhal_pelect_data.pod_state_internal != NEURON_NPE_POD_ST_INIT) || - (ndhal->ndhal_arch.platform_type != NEURON_PLATFORM_TYPE_PDS && - ndhal->ndhal_arch.platform_type != NEURON_PLATFORM_TYPE_MAX && + (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_ELECTION && npe_pod_ctl_is_set(NPE_POD_CTL_RST_SKIP_ELECTION))) { goto done; } @@ -1224,13 +1223,16 @@ int npe_election_exec_on_rst(struct neuron_device *nd, bool reset_successful) goto done; } - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + switch (ndhal->ndhal_arch.platform->pod_discovery) { + case NEURON_POD_DISCOVERY_TYPE_FW_IO: npe_pds_config_init(); - goto done; + break; + case NEURON_POD_DISCOVERY_TYPE_ELECTION: + npe_initiate_election(ndhal_pelect_data.nbr_data_read_timeout); + break; + case NEURON_POD_DISCOVERY_TYPE_NONE: + break; } - - npe_initiate_election(ndhal_pelect_data.nbr_data_read_timeout); done: mutex_unlock(&ndhal_pelect_data.lock); @@ -1279,9 +1281,9 @@ static int npe_get_modal_node_id(enum neuron_ultraserver_mode mode) { int node_id = ndhal_pelect_data.node_id; - // PDS/MAX doesn't change node_id based on mode because nodes id are location based vs. election based - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + // firmware-provided node_ids are (location-based) and don't depend on mode; + // only election-derived node_ids remap when the operating mode changes. + if (ndhal->ndhal_arch.platform->pod_discovery != NEURON_POD_DISCOVERY_TYPE_ELECTION) { return node_id; } @@ -1622,9 +1624,7 @@ int npe_pod_ctrl(struct neuron_device *nd, u32 ctrl, enum neuron_ultraserver_mod } else if (ctrl == NEURON_NPE_POD_CTRL_REQ_POD) { int mark_cnt = ncrwl_range_mark_cnt_get(); - // no election required on PDS/MAX, return success - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + if (ndhal->ndhal_arch.platform->pod_discovery != NEURON_POD_DISCOVERY_TYPE_ELECTION) { ret = 0; goto done; } @@ -1660,13 +1660,12 @@ int npe_pod_ctrl(struct neuron_device *nd, u32 ctrl, enum neuron_ultraserver_mod */ int npe_platform_ready(struct neuron_device *nd, enum neuron_platform_operation_type platform_operation) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_FW_IO) { switch (platform_operation) { case NEURON_PLATFORM_OP_TYPE_DEVOPEN: - if (npe_pod_state_busy()) { + if (npe_pod_state_busy()) { return -EBUSY; - } + } return 0; case NEURON_PLATFORM_OP_TYPE_EXEC: @@ -1675,7 +1674,7 @@ int npe_platform_ready(struct neuron_device *nd, enum neuron_platform_operation_ default: break; } - } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_ULTRASERVER) { + } else if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_ELECTION) { switch (platform_operation) { case NEURON_PLATFORM_OP_TYPE_DEVOPEN: return 0; @@ -1814,13 +1813,12 @@ ssize_t npe_class_node_id_show_data(char *buf, u32 sz) return dhal_sysfs_emit(buf, "busy\n"); } - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_FW_IO) { mode = npe_node_cnt_to_mode(ndhal_pelect_data.node_cnt); - } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_ULTRASERVER) { + } else if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_ELECTION) { mode = npe_node_cnt_to_mode(sz); } else { - pr_err("unexpected platform type %d", ndhal->ndhal_arch.platform_type); + pr_err("unexpected platform type %d", ndhal->ndhal_arch.platform->type); return dhal_sysfs_emit(buf, "invalid\n"); } @@ -1832,8 +1830,7 @@ ssize_t npe_class_node_cnt_show_data(char *buf) { int node_cnt = -1; // node_cnt is currently only returned for PDS/MAX - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_FW_IO) { node_cnt = ndhal_pelect_data.node_cnt; } @@ -1855,10 +1852,9 @@ ssize_t npe_class_server_id_show_data(char *buf, u32 sz) return dhal_sysfs_emit(buf, "0000000000000000\n"); } - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_FW_IO) { mode = npe_node_cnt_to_mode(ndhal_pelect_data.node_cnt); - } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_ULTRASERVER) { + } else if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_ELECTION) { mode = npe_node_cnt_to_mode(sz); } @@ -1929,6 +1925,8 @@ void npe_pds_config_init(void) int partition_sz; int server_id; int rack_id; + int max_racks_per_pod; + int max_servers_per_rack; if (initialized) { return; @@ -1967,7 +1965,7 @@ void npe_pds_config_init(void) } if ((partition_sz == -1) || (instance_sz <= 0)) { - int default_node_cnt = (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) ? 1 : 4; + int default_node_cnt = ndhal->ndhal_arch.platform->default_node_cnt; pr_warn("partition/instance size data is invalid (%d/%d), defaulting to %d node configuration", partition_sz, instance_sz, default_node_cnt); ndhal_pelect_data.node_cnt = default_node_cnt; } else { @@ -2002,22 +2000,18 @@ void npe_pds_config_init(void) // map server/rack to node id. TODO covert to mapping function // - switch ((rack_id << 1) | server_id) { - case 0: - ndhal_pelect_data.node_id = 0; - break; - case 1: - ndhal_pelect_data.node_id = 1; - break; - case 2: - ndhal_pelect_data.node_id = 3; - break; - case 3: - ndhal_pelect_data.node_id = 2; - break; - default: - ndhal_pelect_data.node_id = -1; - break; + if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_MAX) { + max_racks_per_pod = 2; + max_servers_per_rack = 18; + } else { + max_racks_per_pod = 2; + max_servers_per_rack = 2; + } + + if ((rack_id >= 0) && (rack_id < max_racks_per_pod) && (server_id >= 0) && (server_id < max_servers_per_rack)) { + ndhal_pelect_data.node_id = rack_id ? (((max_servers_per_rack * 2) - 1) - server_id) : server_id; + } else { + ndhal_pelect_data.node_id = -1; } done: @@ -2048,3 +2042,4 @@ void npe_cleanup(void) ndhal_pelect_data.pnd[0] = NULL; } } + diff --git a/v3/neuron_pelect.h b/v3/neuron_pelect.h index 0f99af0..ed2b338 100644 --- a/v3/neuron_pelect.h +++ b/v3/neuron_pelect.h @@ -126,3 +126,4 @@ ssize_t npe_class_server_id_show_data(char *buf, u32 sz); ssize_t npe_class_ultraserver_mode_show_data(char *buf); #endif + diff --git a/v3/notific.c b/v3/notific.c index 3c1e82a..293ebdf 100644 --- a/v3/notific.c +++ b/v3/notific.c @@ -130,3 +130,4 @@ int notific_decode_nq_head_reg_access_v3(u64 offset, u8 *nc_id, u32 *nq_type, u8 return -EINVAL; } + diff --git a/v3/notific.h b/v3/notific.h index ca16d99..7ab8982 100644 --- a/v3/notific.h +++ b/v3/notific.h @@ -63,3 +63,4 @@ static inline u64 notific_get_relative_offset_topsp_v3(int ts_idx) int notific_decode_nq_head_reg_access_v3(u64 offset, u8 *nc_id, u32 *nq_type, u8 *instance, bool *is_top_sp); + diff --git a/v3/sdma.h b/v3/sdma.h index 4bd7d78..51b8731 100644 --- a/v3/sdma.h +++ b/v3/sdma.h @@ -111,3 +111,4 @@ static inline int sdma_configure_broadcast_v3(void __iomem *regs_base, int eng_i return 0; } + diff --git a/v4/address_map.h b/v4/address_map.h index 225e914..1642db6 100644 --- a/v4/address_map.h +++ b/v4/address_map.h @@ -227,3 +227,4 @@ #define V4_TPB_ARR_SEQ_QUEUE_PERF_IDLE_CYCLE_CNT_LSB_OFFSET 0x00000000000010ull #endif + diff --git a/v4/neuron_dhal_v4.c b/v4/neuron_dhal_v4.c index cb656da..39d2031 100644 --- a/v4/neuron_dhal_v4.c +++ b/v4/neuron_dhal_v4.c @@ -16,6 +16,7 @@ #include "../neuron_dma.h" #include "../neuron_pci.h" #include "../v3/neuron_pelect.h" +#include "neuron_fw_metrics.h" // TOP SP addresses are sparse on chip adjust to accommodate the table macro @@ -148,60 +149,29 @@ static int ndhal_register_funcs_trn3(void) { return 0; } -static const struct neuron_platform_lookup v4_platform_map[] = { - {"trn3e.24xlarge", NEURON_PLATFORM_TYPE_MAX}, - {"trn3e-dev0.24xlarge", NEURON_PLATFORM_TYPE_MAX}, - {"trn3.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn3-dev0.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn3-dev1.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn3s-es.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn3sn.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn3sn-es.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn3p.48xlarge", NEURON_PLATFORM_TYPE_ULTRASERVER}, - {NULL, NEURON_PLATFORM_TYPE_INVALID}, +static const uint32_t v4_nnq_h2d_id_to_eng_id[] = { + V4_D2H_0_IDX, V4_H2D_0_IDX, V4_D2H_1_IDX, V4_H2D_1_IDX }; -static enum neuron_platform_type ndhal_platform_type_v4(void) +static int nnq_get_nq_h2d_id_v4(uint32_t h2d_eng_id) { - enum neuron_platform_type platform_type; - - platform_type = narch_detect_platform_type(v4_platform_map); - - if (narch_is_qemu() || narch_is_emu()) - platform_type = NEURON_PLATFORM_TYPE_STD; - - return platform_type; -} - -static bool ndhal_instance_type_3xl(void) -{ - static const char *const trn3_3xl_instance_names[] = { - "trn3pd98.3xlarge", - "trn3s-es.3xlarge", - "trn3sn.3xlarge", - "trn3sn-es.3xlarge", - }; - static bool instance_type_is_3xl = false; - char buf[128]; int i; - - if (narch_get_instance_type_name(buf, sizeof(buf))) goto done; - for (i = 0; i < sizeof(trn3_3xl_instance_names) / sizeof(*trn3_3xl_instance_names); i++) { - if (strncmp(buf, trn3_3xl_instance_names[i], strlen(trn3_3xl_instance_names[i])) == 0) { - instance_type_is_3xl = true; - break; - } + for (i = 0; i < ARRAY_SIZE(v4_nnq_h2d_id_to_eng_id); i++) { + if (v4_nnq_h2d_id_to_eng_id[i] == h2d_eng_id) + return i; } + return -1; +} -done: - return instance_type_is_3xl; +static uint32_t nnq_get_h2d_id_v4(int nq_h2d_id) +{ + return v4_nnq_h2d_id_to_eng_id[nq_h2d_id]; } /** - * ndmar_ctx_queue_bit_v4() - map a V4 H2D queue to a dense bitmap index + * * ndmar_ctx_queue_bit_v4() - map a V4 H2D queue to a dense bitmap index * @h2d_eng_id: DMA engine id in the V4 H2D/D2H engine range (128 to 131) * @qid: DMA queue id within the engine - * * Return bitmap bit index for the queue. */ static int ndmar_ctx_queue_bit_v4(uint32_t h2d_eng_id, uint32_t qid) @@ -310,35 +280,233 @@ static const u32 v4_torus_routing_id_to_user_id[] = { 1, 2, 5, 6, 13, 14, 9, 10 }; -// map routing id to user id for trn2pds instance type. -// the only hard rule this map needs to follow is -// rid (i*2) and rid (i*2)+1 map to did (n*2) and did (n*2)+1 -// since rid (i*2) and rid (i*2)+1 are on the same JBOG. -static const u32 v4_pds_routing_id_to_user_id[] = { - 0, 1, - 2, 3, - 4, 5, - 6, 7, - 8, 9, - 10, 11, - 12, 13, - 14, 15 }; - -static const u32 v4_max_routing_id_to_user_id[] = { - 0, 1, 2, 3 }; - #define V4_ROUTING_ID_TBL_SZ (sizeof(v4_torus_routing_id_to_user_id) / sizeof(v4_torus_routing_id_to_user_id[0])) -#define V4_MAX_ROUTING_ID_TBL_SZ (sizeof(v4_max_routing_id_to_user_id) / sizeof(v4_max_routing_id_to_user_id[0])) + +static const struct neuron_platform_desc v4_platform_std = { + .type = NEURON_PLATFORM_TYPE_STD, + .family = NEURON_PLATFORM_FAMILY_NONE, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_TABLE, .table = { v4_torus_routing_id_to_user_id, V4_ROUTING_ID_TBL_SZ } }, + .device_count = 16, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_NONE } }, + .nc_efa_bdf_tbl = { .bdfs = trn3pd_nc_efa_bdf_tbl, .count = sizeof(trn3pd_nc_efa_bdf_tbl) / sizeof(trn3pd_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 8, + .num_chips_per_sled = 2, +}; + +static const char *const v4_ultraserver_instance_names[] = { + "trn3p.48xlarge", + NULL, +}; + +static const char *const v4_pds_v2_switch_instance_names[] = { + "trn3n.48xlarge", + NULL, +}; + +static const char *const v4_pds_v1_switch_instance_names[] = { + "trn3.48xlarge", + "trn3-dev0.48xlarge", + "trn3-dev1.48xlarge", + "trn3-es.48xlarge", + "trn3s-es.48xlarge", + NULL, +}; + +static const char *const v4_max_v2_switch_instance_names[] = { + "trn3e.24xlarge", + "trn3en.24xlarge", + NULL, +}; + +static const char *const v4_max_v1_switch_instance_names[] = { + "trn3e-dev0.24xlarge", + "trn3e-es.24xlarge", + NULL, +}; + +// Simulation-only instance name for simulated (QEMU/inkling) MAX compute +// sleds. Deliberately not a real instance name: platform capabilities are +// consumed from the driver's platform descriptor (sysfs), not derived from +// DMI, so a simulation does not need to impersonate the instance it models. +static const char *const v4_qemu_max_instance_names[] = { + "qemu-trn3max", + NULL, +}; + +static const char *const v4_3xl_instance_names[] = { + "trn3pd98.3xlarge", + "trn3s-es.3xlarge", + "trn3.3xlarge", + "trn3-es.3xlarge", + "trn3n.3xlarge", + NULL, +}; + +static const struct neuron_platform_desc v4_platform_table[] = { + { + .instance_names = v4_ultraserver_instance_names, + .type = NEURON_PLATFORM_TYPE_ULTRASERVER, + .family = NEURON_PLATFORM_FAMILY_TRN2_ULTRASERVER, /* trn3p.48xlarge is a dev platform on the deprecation path; use TRN2_ULTRASERVER as it is the publicly documented family for the trn2u topology */ + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_TABLE, .table = { v4_torus_routing_id_to_user_id, V4_ROUTING_ID_TBL_SZ } }, + .device_count = 16, + .default_node_cnt = 4, + .pod_type = NEURON_POD_TYPE_P2P, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_ELECTION, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_TORUS }, .inter_rack = { .topology = NEURON_INTER_RACK_TOPOLOGY_P2P } }, + .nc_efa_bdf_tbl = { .bdfs = trn3pd_nc_efa_bdf_tbl, .count = sizeof(trn3pd_nc_efa_bdf_tbl) / sizeof(trn3pd_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 8, + .num_chips_per_sled = 2, + }, + { + .instance_names = v4_pds_v2_switch_instance_names, + .type = NEURON_PLATFORM_TYPE_PDS, + .family = NEURON_PLATFORM_FAMILY_TRN3_GEN1_ULTRASERVER, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 16, + .default_node_cnt = 4, + .pod_type = NEURON_POD_TYPE_SWITCH, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_FW_IO, + .max_bars = 1, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_SWITCH, .switch_type = NEURON_SWITCH_TYPE_V2 }, .inter_rack = { .topology = NEURON_INTER_RACK_TOPOLOGY_P2P } }, + .nc_efa_bdf_tbl = { .bdfs = trn3pds_nc_efa_bdf_tbl, .count = sizeof(trn3pds_nc_efa_bdf_tbl) / sizeof(trn3pds_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 8, + .num_chips_per_sled = 2, + }, + { + .instance_names = v4_pds_v1_switch_instance_names, + .type = NEURON_PLATFORM_TYPE_PDS, + .family = NEURON_PLATFORM_FAMILY_TRN3_GEN1_ULTRASERVER, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 16, + .default_node_cnt = 4, + .pod_type = NEURON_POD_TYPE_SWITCH, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_FW_IO, + .max_bars = 1, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_SWITCH, .switch_type = NEURON_SWITCH_TYPE_V1 }, .inter_rack = { .topology = NEURON_INTER_RACK_TOPOLOGY_P2P } }, + .nc_efa_bdf_tbl = { .bdfs = trn3pds_nc_efa_bdf_tbl, .count = sizeof(trn3pds_nc_efa_bdf_tbl) / sizeof(trn3pds_nc_efa_bdf_tbl[0]) }, + .num_sleds_per_server = 8, + .num_chips_per_sled = 2, + }, + { + .instance_names = v4_max_v2_switch_instance_names, + .type = NEURON_PLATFORM_TYPE_MAX, + .family = NEURON_PLATFORM_FAMILY_TRN3_GEN2_ULTRASERVER, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 4, + .default_node_cnt = 1, + .pod_type = NEURON_POD_TYPE_SWITCH, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_FW_IO, + .max_bars = 2, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_SWITCH, .switch_type = NEURON_SWITCH_TYPE_V2 }, .inter_rack = { .topology = NEURON_INTER_RACK_TOPOLOGY_P2P } }, + .num_sleds_per_server = 1, + .num_chips_per_sled = 4, + }, + { + .instance_names = v4_max_v1_switch_instance_names, + .type = NEURON_PLATFORM_TYPE_MAX, + .family = NEURON_PLATFORM_FAMILY_TRN3_GEN2_ULTRASERVER, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 4, + .default_node_cnt = 1, + .pod_type = NEURON_POD_TYPE_SWITCH, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_FW_IO, + .max_bars = 2, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_SWITCH, .switch_type = NEURON_SWITCH_TYPE_V1 }, .inter_rack = { .topology = NEURON_INTER_RACK_TOPOLOGY_P2P } }, + .num_sleds_per_server = 1, + .num_chips_per_sled = 4, + }, + { + .instance_names = v4_3xl_instance_names, + .type = NEURON_PLATFORM_TYPE_STD, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 1, + .default_node_cnt = 0, + .pod_type = NEURON_POD_TYPE_NONE, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .num_sleds_per_server = 1, + .num_chips_per_sled = 1, + }, + { + // Simulated (QEMU/inkling) MAX compute sled. A simulated compute + // sled is one QEMU guest standing in for the sled's integrated host + // CPU, so a guest presents like the physical single-sled MAX + // instances above: same SEQUENTIAL routing over the sled's four + // MLAs. This entry differs from the physical MAX descriptors only + // where the device model falls short of real FW today: + // - pod_discovery NONE: the model implements no FW_IO pod + // registers (serial number, partition size, reservation id) yet. + // - max_bars 0: no switch BAR-info emulation yet. + // Both are staged, not permanent: they follow as soon as the device + // model grows the corresponding FW_IO surface. The simulation-only + // instance name cannot appear on real hardware, and the entry is + // only selectable from a simulated instance (see + // ndhal_select_platform_v4). + .instance_names = v4_qemu_max_instance_names, + .type = NEURON_PLATFORM_TYPE_MAX, + .family = NEURON_PLATFORM_FAMILY_TRN3_GEN2_ULTRASERVER, + .routing_config = { .type = NEURON_PLATFORM_ROUTING_TYPE_SEQUENTIAL }, + .device_count = 4, + .default_node_cnt = 1, + .pod_type = NEURON_POD_TYPE_SWITCH, + .pod_discovery = NEURON_POD_DISCOVERY_TYPE_NONE, + .max_bars = 0, + .interconnect = { .intra_rack = { .topology = NEURON_INTRA_RACK_TOPOLOGY_SWITCH, .switch_type = NEURON_SWITCH_TYPE_V1 }, .inter_rack = { .topology = NEURON_INTER_RACK_TOPOLOGY_P2P } }, + .num_sleds_per_server = 1, + .num_chips_per_sled = 4, + }, + { } +}; + +static const struct neuron_platform_desc *ndhal_select_platform_v4(void) +{ + const struct neuron_platform_desc *platform; + + platform = nplatform_get_platform_desc(v4_platform_table); + + // QEMU/emulator: a simulated system opts in to a platform descriptor by + // publishing the simulation-only instance name. A simulated system + // publishing anything else -- in particular the real instance names + // every pre-existing simulated system publishes -- presents as STD + // exactly as before, rather than matching a physical descriptor whose + // probe requirements the simulation may not satisfy. + if (narch_is_qemu() || narch_is_emu()) { + if (platform && platform->instance_names == v4_qemu_max_instance_names) + return platform; + return &v4_platform_std; + } + + return platform ? platform : &v4_platform_std; +} static u32 neuron_pci_routing_id_to_user_id(u32 routing_id) { - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS) { - return v4_pds_routing_id_to_user_id[ routing_id % V4_ROUTING_ID_TBL_SZ]; + u32 index = routing_id % ndhal->ndhal_arch.platform->device_count; + + if (ndhal->ndhal_arch.platform->routing_config.type == NEURON_PLATFORM_ROUTING_TYPE_TABLE) { + return ndhal->ndhal_arch.platform->routing_config.table.map[index]; + } + return index; +} + +static bool ndhal_instance_type_3xl(void) +{ + char buf[128]; + int i; + + if (narch_get_instance_type_name(buf, sizeof(buf))) { + return false; } - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { - return v4_max_routing_id_to_user_id[ routing_id % V4_MAX_ROUTING_ID_TBL_SZ]; + for (i = 0; v4_3xl_instance_names[i]; i++) { + if (strncmp(buf, v4_3xl_instance_names[i], strlen(v4_3xl_instance_names[i])) == 0) + return true; } - return v4_torus_routing_id_to_user_id[routing_id % V4_ROUTING_ID_TBL_SZ]; + return false; } /** @@ -375,8 +543,7 @@ static int neuron_pci_get_device_id_v4(struct neuron_device *nd, struct pci_dev return -ENODEV; } - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + if (ndhal->ndhal_arch.platform->interconnect.intra_rack.topology == NEURON_INTRA_RACK_TOPOLOGY_SWITCH) { int server_id; ret = fw_io_server_info_read(nd->npdev.bar0, &server_id, NULL); @@ -457,19 +624,66 @@ static int ncdev_logical_to_physical_nc_map_v4(struct neuron_ioctl_nc_map *map, return 0; } +#define NC_MAPPING_MAX_CORE_COUNT_MAX_V4 32 +static const struct neuron_ioctl_nc_map_entry nc_mapping_v0_seng_swap_max[] = { + { .device_id = 0, .device_nc_idx = 2 }, { .device_id = 0, .device_nc_idx = 3 }, { .device_id = 0, .device_nc_idx = 0 }, { .device_id = 0, .device_nc_idx = 1 }, { .device_id = 0, .device_nc_idx = 4 }, { .device_id = 0, .device_nc_idx = 5 }, { .device_id = 0, .device_nc_idx = 6 }, { .device_id = 0, .device_nc_idx = 7 }, // ND0 + { .device_id = 1, .device_nc_idx = 2 }, { .device_id = 1, .device_nc_idx = 3 }, { .device_id = 1, .device_nc_idx = 0 }, { .device_id = 1, .device_nc_idx = 1 }, { .device_id = 1, .device_nc_idx = 4 }, { .device_id = 1, .device_nc_idx = 5 }, { .device_id = 1, .device_nc_idx = 6 }, { .device_id = 1, .device_nc_idx = 7 }, // ND1 + { .device_id = 2, .device_nc_idx = 4 }, { .device_id = 2, .device_nc_idx = 5 }, { .device_id = 2, .device_nc_idx = 6 }, { .device_id = 2, .device_nc_idx = 7 }, { .device_id = 2, .device_nc_idx = 2 }, { .device_id = 2, .device_nc_idx = 3 }, { .device_id = 2, .device_nc_idx = 0 }, { .device_id = 2, .device_nc_idx = 1 }, // ND2 + { .device_id = 3, .device_nc_idx = 4 }, { .device_id = 3, .device_nc_idx = 5 }, { .device_id = 3, .device_nc_idx = 6 }, { .device_id = 3, .device_nc_idx = 7 }, { .device_id = 3, .device_nc_idx = 2 }, { .device_id = 3, .device_nc_idx = 3 }, { .device_id = 3, .device_nc_idx = 0 }, { .device_id = 3, .device_nc_idx = 1 }, // ND3 +}; + +#define NC_MAPPING_V0_SENG_SWAP_MAX_SIZE (sizeof(nc_mapping_v0_seng_swap_max) / sizeof(nc_mapping_v0_seng_swap_max[0])) +static_assert((NC_MAPPING_V0_SENG_SWAP_MAX_SIZE == NC_MAPPING_MAX_CORE_COUNT_MAX_V4) && (NC_MAPPING_V0_SENG_SWAP_MAX_SIZE <= NEURON_NC_MAP_MAX_ENTRIES)); +static int ncdev_logical_to_physical_nc_map_max_v4(struct neuron_ioctl_nc_map *map, uint32_t max_num_entries, enum neuron_ioctl_nc_mapping_type version) +{ + uint32_t entry_idx; + uint32_t entries_to_copy = (max_num_entries < NC_MAPPING_MAX_CORE_COUNT_MAX_V4) ? max_num_entries : NC_MAPPING_MAX_CORE_COUNT_MAX_V4; + const struct neuron_ioctl_nc_map_entry *mapping; + + if (version != NEURON_IOCTL_NC_MAPPING_TYPE_V0) { + pr_err("Unsupported Neuron Core Mapping verion %u for v4 arch", version); + return -EINVAL; + } + mapping = nc_mapping_v0_seng_swap_max; + + for (entry_idx = 0; entry_idx < entries_to_copy; entry_idx++) { + uint32_t core_idx = entry_idx; + WARN_ONCE(core_idx >= NC_MAPPING_MAX_CORE_COUNT_MAX_V4, "core_idx %d > max core count %d", core_idx, NC_MAPPING_MAX_CORE_COUNT_MAX_V4); + map->mappings[entry_idx] = mapping[core_idx]; + } + map->num_entries = entries_to_copy; + + return 0; +} + static void perf_update_hbm_7200_supported_v4(struct neuron_device *nd) { nd->supports_hbm_7200 = 0; return; } +/** + * ndma_get_engines_with_host_connectivity - get DMA engines for a particular HBM index which have host connectivity + * V4 - unlike V3, all DMA engines for an HBM have host connectivity, so use all 32. + */ +static void ndma_get_engines_with_host_connectivity_v4(u32 hbm_index, u32 engines[NUM_DMA_ENG_PER_DEVICE], u32 *num_engines) +{ + const int num_dma_engines_per_hbm = 32; + const int offset = num_dma_engines_per_hbm * hbm_index; + + int i = 0; + for (i = 0; i < num_dma_engines_per_hbm; i++) { + engines[i] = offset + i; + } + *num_engines = num_dma_engines_per_hbm; +} + static int neuron_pci_device_id_to_rid_map_v4(uint32_t *count, uint32_t *did_to_rid_map) { int i; for (i = 0; i < total_neuron_devices; i++) { u32 routing_id; - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS || - ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + if (ndhal->ndhal_arch.platform->interconnect.intra_rack.topology == NEURON_INTRA_RACK_TOPOLOGY_SWITCH) { routing_id = i + ndhal->ndhal_arch.server_id * total_neuron_devices; } else { routing_id = i; @@ -495,19 +709,25 @@ int ndhal_register_funcs_v4(void) { return -EINVAL; } - ndhal->ndhal_arch.platform_type = ndhal_platform_type_v4(); + ndhal->ndhal_arch.platform = ndhal_select_platform_v4(); ndhal->ndhal_fw_io.new_readless_read_min_api_version = 6; + fw_metrics_init_reader_v4(); ndhal->ndhal_pci.neuron_pci_get_device_id = neuron_pci_get_device_id_v4; ndhal->ndhal_pci.neuron_pci_device_id_to_rid_map = neuron_pci_device_id_to_rid_map_v4; ndhal->ndhal_npe.npe_neighbor_eng_ids = npe_neighbor_eng_ids_v4; ndhal->ndhal_mpset.mpset_set_dram_and_mpset_info = mpset_set_dram_and_mpset_info_v4; ndhal->ndhal_mmap.dm_mmap_special = dm_mmap_special_v4; ndhal->ndhal_mmap.mmap_get_bar4_offset = mmap_get_bar4_offset_v4; + ndhal->ndhal_ndma.ndma_get_engines_with_host_connectivity = ndma_get_engines_with_host_connectivity_v4; ndhal->ndhal_ndmar.ndmar_ctx_queue_bit = ndmar_ctx_queue_bit_v4; ndhal->ndhal_ndmar.ndmar_ctx_queue_from_bit = ndmar_ctx_queue_from_bit_v4; + ndhal->ndhal_nq.nnq_get_nq_h2d_id = nnq_get_nq_h2d_id_v4; + ndhal->ndhal_nq.nnq_get_h2d_id = nnq_get_h2d_id_v4; ndhal->ndhal_cdev.ncdev_mem_regions = ncdev_mem_regions_v4; ndhal->ndhal_perf.perf_update_hbm_7200_supported = perf_update_hbm_7200_supported_v4; ndhal->ndhal_sysfs_metrics.health_status_enabled = true; + ndhal->ndhal_address_map.hbm_active_size = V4_HBM_ACTIVE_SIZE; + ndhal->ndhal_address_map.supported_lnc_degrees = BIT(1) | BIT(2); if (narch_is_emu()) { // Temporarily disable resets on emulation until support is ready @@ -518,18 +738,18 @@ int ndhal_register_funcs_v4(void) { // TODO initialization needs refactoring because V4 is piggybacking on V3 // which risks double calling any hal init functions // - if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_ULTRASERVER) { + if (ndhal->ndhal_arch.platform->pod_discovery == NEURON_POD_DISCOVERY_TYPE_ELECTION) { ret = npe_init(); if (ret) { pr_err("failed to initialize pod election on V4\n"); return ret; } - } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS) { + } else if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_PDS) { //TODO PDS ndhal->ndhal_cdev.ncdev_logical_to_physical_nc_map = ncdev_logical_to_physical_nc_map_v4; - } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { - ndhal->ndhal_cdev.ncdev_logical_to_physical_nc_map = ncdev_logical_to_physical_nc_map_v4; - } + } else if (ndhal->ndhal_arch.platform->type == NEURON_PLATFORM_TYPE_MAX) { + ndhal->ndhal_cdev.ncdev_logical_to_physical_nc_map = ncdev_logical_to_physical_nc_map_max_v4; + } switch (ndhal->pci_device_id) { case TRN3_DEVICE_ID0: @@ -547,3 +767,4 @@ int ndhal_register_funcs_v4(void) { return ret; } + diff --git a/v4/neuron_fw_metrics.c b/v4/neuron_fw_metrics.c new file mode 100644 index 0000000..b86fb08 --- /dev/null +++ b/v4/neuron_fw_metrics.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + * + * v4-specific firmware metrics: the supported misc-ram version and the payload + * parsers (shared helpers in ../neuron_fw_metrics_common.h). + */ + +#include + +#include "../neuron_dhal.h" +#include "neuron_fw_metrics.h" + +// no payload schema is adopted on v4 yet +static int fw_metrics_parse_payload_v4(struct neuron_device *nd, u16 type_id, const u8 *payload, u32 length) +{ + return -ENOTSUPP; +} + +void fw_metrics_init_reader_v4(void) +{ + ndhal->ndhal_fw_metrics.max_misc_ram_version = V4_FW_METRICS_MAX_MISC_RAM_VERSION; + ndhal->ndhal_fw_metrics.parse_payload = fw_metrics_parse_payload_v4; +} + diff --git a/v4/neuron_fw_metrics.h b/v4/neuron_fw_metrics.h new file mode 100644 index 0000000..9739fb7 --- /dev/null +++ b/v4/neuron_fw_metrics.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2026, Amazon.com, Inc. or its affiliates. All Rights Reserved + * + * v4-specific firmware metrics; shared code lives in ../neuron_fw_metrics_common.h. + */ + +#ifndef NEURON_FW_METRICS_V4_H +#define NEURON_FW_METRICS_V4_H + +// highest misc-ram version whose metrics this driver parses on v4 (sent via SET_MAX_VERSION) +#define V4_FW_METRICS_MAX_MISC_RAM_VERSION 0x15 + +void fw_metrics_init_reader_v4(void); + +#endif + diff --git a/vc/neuron_dhal_vc.c b/vc/neuron_dhal_vc.c index e550d48..41eb32c 100644 --- a/vc/neuron_dhal_vc.c +++ b/vc/neuron_dhal_vc.c @@ -149,3 +149,4 @@ int ndhal_register_funcs_vc(void) ndhal->ndhal_tpb.pe_get_fast_wl_cycle_cnt = ntpb_pe_get_fast_wl_cycle_cnt_vc; return 0; } +