diff options
7 files changed, 236 insertions, 66 deletions
diff --git a/drivers/net/ethernet/microchip/lan966x/Kconfig b/drivers/net/ethernet/microchip/lan966x/Kconfig index 49e1464a4313..b7ae5ce7d3f7 100644 --- a/drivers/net/ethernet/microchip/lan966x/Kconfig +++ b/drivers/net/ethernet/microchip/lan966x/Kconfig @@ -7,5 +7,6 @@ config LAN966X_SWITCH depends on BRIDGE || BRIDGE=n select PHYLINK select PACKING + select PAGE_POOL help This driver supports the Lan966x network switch device. diff --git a/drivers/net/ethernet/microchip/lan966x/Makefile b/drivers/net/ethernet/microchip/lan966x/Makefile index 962f7c5f9e7d..251a7d561d63 100644 --- a/drivers/net/ethernet/microchip/lan966x/Makefile +++ b/drivers/net/ethernet/microchip/lan966x/Makefile @@ -11,4 +11,5 @@ lan966x-switch-objs := lan966x_main.o lan966x_phylink.o lan966x_port.o \ lan966x_ptp.o lan966x_fdma.o lan966x_lag.o \ lan966x_tc.o lan966x_mqprio.o lan966x_taprio.o \ lan966x_tbf.o lan966x_cbs.o lan966x_ets.o \ - lan966x_tc_matchall.o lan966x_police.o lan966x_mirror.o + lan966x_tc_matchall.o lan966x_police.o lan966x_mirror.o \ + lan966x_xdp.o diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c index e6948939ccc2..5fbbd479cfb0 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c @@ -10,50 +10,39 @@ static int lan966x_fdma_channel_active(struct lan966x *lan966x) static struct page *lan966x_fdma_rx_alloc_page(struct lan966x_rx *rx, struct lan966x_db *db) { - struct lan966x *lan966x = rx->lan966x; - dma_addr_t dma_addr; struct page *page; - page = dev_alloc_pages(rx->page_order); + page = page_pool_dev_alloc_pages(rx->page_pool); if (unlikely(!page)) return NULL; - dma_addr = dma_map_page(lan966x->dev, page, 0, - PAGE_SIZE << rx->page_order, - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(lan966x->dev, dma_addr))) - goto free_page; - - db->dataptr = dma_addr; + db->dataptr = page_pool_get_dma_addr(page); return page; - -free_page: - __free_pages(page, rx->page_order); - return NULL; } static void lan966x_fdma_rx_free_pages(struct lan966x_rx *rx) { - struct lan966x *lan966x = rx->lan966x; - struct lan966x_rx_dcb *dcb; - struct lan966x_db *db; int i, j; for (i = 0; i < FDMA_DCB_MAX; ++i) { - dcb = &rx->dcbs[i]; - - for (j = 0; j < FDMA_RX_DCB_MAX_DBS; ++j) { - db = &dcb->db[j]; - dma_unmap_single(lan966x->dev, - (dma_addr_t)db->dataptr, - PAGE_SIZE << rx->page_order, - DMA_FROM_DEVICE); - __free_pages(rx->page[i][j], rx->page_order); - } + for (j = 0; j < FDMA_RX_DCB_MAX_DBS; ++j) + page_pool_put_full_page(rx->page_pool, + rx->page[i][j], false); } } +static void lan966x_fdma_rx_free_page(struct lan966x_rx *rx) +{ + struct page *page; + + page = rx->page[rx->dcb_index][rx->db_index]; + if (unlikely(!page)) + return; + + page_pool_recycle_direct(rx->page_pool, page); +} + static void lan966x_fdma_rx_add_dcb(struct lan966x_rx *rx, struct lan966x_rx_dcb *dcb, u64 nextptr) @@ -73,6 +62,25 @@ static void lan966x_fdma_rx_add_dcb(struct lan966x_rx *rx, rx->last_entry = dcb; } +static int lan966x_fdma_rx_alloc_page_pool(struct lan966x_rx *rx) +{ + struct lan966x *lan966x = rx->lan966x; + struct page_pool_params pp_params = { + .order = rx->page_order, + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .pool_size = FDMA_DCB_MAX, + .nid = NUMA_NO_NODE, + .dev = lan966x->dev, + .dma_dir = DMA_FROM_DEVICE, + .offset = 0, + .max_len = rx->max_mtu - + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), + }; + + rx->page_pool = page_pool_create(&pp_params); + return PTR_ERR_OR_ZERO(rx->page_pool); +} + static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx) { struct lan966x *lan966x = rx->lan966x; @@ -82,6 +90,9 @@ static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx) int i, j; int size; + if (lan966x_fdma_rx_alloc_page_pool(rx)) + return PTR_ERR(rx->page_pool); + /* calculate how many pages are needed to allocate the dcbs */ size = sizeof(struct lan966x_rx_dcb) * FDMA_DCB_MAX; size = ALIGN(size, PAGE_SIZE); @@ -116,6 +127,12 @@ static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx) return 0; } +static void lan966x_fdma_rx_advance_dcb(struct lan966x_rx *rx) +{ + rx->dcb_index++; + rx->dcb_index &= FDMA_DCB_MAX - 1; +} + static void lan966x_fdma_rx_free(struct lan966x_rx *rx) { struct lan966x *lan966x = rx->lan966x; @@ -403,40 +420,58 @@ static bool lan966x_fdma_rx_more_frames(struct lan966x_rx *rx) return true; } -static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx) +static int lan966x_fdma_rx_check_frame(struct lan966x_rx *rx, u64 *src_port) { struct lan966x *lan966x = rx->lan966x; - u64 src_port, timestamp; + struct lan966x_port *port; struct lan966x_db *db; - struct sk_buff *skb; struct page *page; - /* Get the received frame and unmap it */ db = &rx->dcbs[rx->dcb_index].db[rx->db_index]; page = rx->page[rx->dcb_index][rx->db_index]; + if (unlikely(!page)) + return FDMA_ERROR; dma_sync_single_for_cpu(lan966x->dev, (dma_addr_t)db->dataptr, FDMA_DCB_STATUS_BLOCKL(db->status), DMA_FROM_DEVICE); + lan966x_ifh_get_src_port(page_address(page), src_port); + if (WARN_ON(*src_port >= lan966x->num_phys_ports)) + return FDMA_ERROR; + + port = lan966x->ports[*src_port]; + if (!lan966x_xdp_port_present(port)) + return FDMA_PASS; + + return lan966x_xdp_run(port, page, FDMA_DCB_STATUS_BLOCKL(db->status)); +} + +static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx, + u64 src_port) +{ + struct lan966x *lan966x = rx->lan966x; + struct lan966x_db *db; + struct sk_buff *skb; + struct page *page; + u64 timestamp; + + /* Get the received frame and unmap it */ + db = &rx->dcbs[rx->dcb_index].db[rx->db_index]; + page = rx->page[rx->dcb_index][rx->db_index]; + skb = build_skb(page_address(page), PAGE_SIZE << rx->page_order); if (unlikely(!skb)) - goto unmap_page; + goto free_page; + + skb_mark_for_recycle(skb); skb_put(skb, FDMA_DCB_STATUS_BLOCKL(db->status)); - lan966x_ifh_get_src_port(skb->data, &src_port); lan966x_ifh_get_timestamp(skb->data, ×tamp); - if (WARN_ON(src_port >= lan966x->num_phys_ports)) - goto free_skb; - - dma_unmap_single_attrs(lan966x->dev, (dma_addr_t)db->dataptr, - PAGE_SIZE << rx->page_order, DMA_FROM_DEVICE, - DMA_ATTR_SKIP_CPU_SYNC); - skb->dev = lan966x->ports[src_port]->dev; - skb_pull(skb, IFH_LEN * sizeof(u32)); + skb_pull(skb, IFH_LEN_BYTES); if (likely(!(skb->dev->features & NETIF_F_RXFCS))) skb_trim(skb, skb->len - ETH_FCS_LEN); @@ -457,13 +492,8 @@ static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx) return skb; -free_skb: - kfree_skb(skb); -unmap_page: - dma_unmap_single_attrs(lan966x->dev, (dma_addr_t)db->dataptr, - PAGE_SIZE << rx->page_order, DMA_FROM_DEVICE, - DMA_ATTR_SKIP_CPU_SYNC); - __free_pages(page, rx->page_order); +free_page: + page_pool_recycle_direct(rx->page_pool, page); return NULL; } @@ -478,6 +508,7 @@ static int lan966x_fdma_napi_poll(struct napi_struct *napi, int weight) struct sk_buff *skb; struct page *page; int counter = 0; + u64 src_port; u64 nextptr; lan966x_fdma_tx_clear_buf(lan966x, weight); @@ -487,19 +518,30 @@ static int lan966x_fdma_napi_poll(struct napi_struct *napi, int weight) if (!lan966x_fdma_rx_more_frames(rx)) break; - skb = lan966x_fdma_rx_get_frame(rx); + counter++; - rx->page[rx->dcb_index][rx->db_index] = NULL; - rx->dcb_index++; - rx->dcb_index &= FDMA_DCB_MAX - 1; + switch (lan966x_fdma_rx_check_frame(rx, &src_port)) { + case FDMA_PASS: + break; + case FDMA_ERROR: + lan966x_fdma_rx_free_page(rx); + lan966x_fdma_rx_advance_dcb(rx); + goto allocate_new; + case FDMA_DROP: + lan966x_fdma_rx_free_page(rx); + lan966x_fdma_rx_advance_dcb(rx); + continue; + } + skb = lan966x_fdma_rx_get_frame(rx, src_port); + lan966x_fdma_rx_advance_dcb(rx); if (!skb) - break; + goto allocate_new; napi_gro_receive(&lan966x->napi, skb); - counter++; } +allocate_new: /* Allocate new pages and map them */ while (dcb_reload != rx->dcb_index) { db = &rx->dcbs[dcb_reload].db[rx->db_index]; @@ -592,7 +634,7 @@ int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev) } /* skb processing */ - needed_headroom = max_t(int, IFH_LEN * sizeof(u32) - skb_headroom(skb), 0); + needed_headroom = max_t(int, IFH_LEN_BYTES - skb_headroom(skb), 0); needed_tailroom = max_t(int, ETH_FCS_LEN - skb_tailroom(skb), 0); if (needed_headroom || needed_tailroom || skb_header_cloned(skb)) { err = pskb_expand_head(skb, needed_headroom, needed_tailroom, @@ -605,8 +647,8 @@ int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev) } skb_tx_timestamp(skb); - skb_push(skb, IFH_LEN * sizeof(u32)); - memcpy(skb->data, ifh, IFH_LEN * sizeof(u32)); + skb_push(skb, IFH_LEN_BYTES); + memcpy(skb->data, ifh, IFH_LEN_BYTES); skb_put(skb, 4); dma_addr = dma_map_single(lan966x->dev, skb->data, skb->len, @@ -696,6 +738,7 @@ static int lan966x_qsys_sw_status(struct lan966x *lan966x) static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) { + struct page_pool *page_pool; dma_addr_t rx_dma; void *rx_dcbs; u32 size; @@ -704,6 +747,7 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) /* Store these for later to free them */ rx_dma = lan966x->rx.dma; rx_dcbs = lan966x->rx.dcbs; + page_pool = lan966x->rx.page_pool; napi_synchronize(&lan966x->napi); napi_disable(&lan966x->napi); @@ -712,6 +756,7 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) lan966x_fdma_rx_disable(&lan966x->rx); lan966x_fdma_rx_free_pages(&lan966x->rx); lan966x->rx.page_order = round_up(new_mtu, PAGE_SIZE) / PAGE_SIZE - 1; + lan966x->rx.max_mtu = new_mtu; err = lan966x_fdma_rx_alloc(&lan966x->rx); if (err) goto restore; @@ -721,11 +766,14 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) size = ALIGN(size, PAGE_SIZE); dma_free_coherent(lan966x->dev, size, rx_dcbs, rx_dma); + page_pool_destroy(page_pool); + lan966x_fdma_wakeup_netdev(lan966x); napi_enable(&lan966x->napi); return err; restore: + lan966x->rx.page_pool = page_pool; lan966x->rx.dma = rx_dma; lan966x->rx.dcbs = rx_dcbs; lan966x_fdma_rx_start(&lan966x->rx); @@ -733,19 +781,22 @@ restore: return err; } +static int lan966x_fdma_get_max_frame(struct lan966x *lan966x) +{ + return lan966x_fdma_get_max_mtu(lan966x) + + IFH_LEN_BYTES + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + VLAN_HLEN * 2; +} + int lan966x_fdma_change_mtu(struct lan966x *lan966x) { int max_mtu; int err; u32 val; - max_mtu = lan966x_fdma_get_max_mtu(lan966x); - max_mtu += IFH_LEN * sizeof(u32); - max_mtu += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); - max_mtu += VLAN_HLEN * 2; - - if (round_up(max_mtu, PAGE_SIZE) / PAGE_SIZE - 1 == - lan966x->rx.page_order) + max_mtu = lan966x_fdma_get_max_frame(lan966x); + if (max_mtu == lan966x->rx.max_mtu) return 0; /* Disable the CPU port */ @@ -800,6 +851,7 @@ int lan966x_fdma_init(struct lan966x *lan966x) lan966x->rx.lan966x = lan966x; lan966x->rx.channel_id = FDMA_XTR_CHANNEL; + lan966x->rx.max_mtu = lan966x_fdma_get_max_frame(lan966x); lan966x->tx.lan966x = lan966x; lan966x->tx.channel_id = FDMA_INJ_CHANNEL; lan966x->tx.last_in_use = -1; @@ -832,5 +884,6 @@ void lan966x_fdma_deinit(struct lan966x *lan966x) lan966x_fdma_rx_free_pages(&lan966x->rx); lan966x_fdma_rx_free(&lan966x->rx); + page_pool_destroy(lan966x->rx.page_pool); lan966x_fdma_tx_free(&lan966x->tx); } diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ifh.h b/drivers/net/ethernet/microchip/lan966x/lan966x_ifh.h index ca3314789d18..f3b1e0d31826 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_ifh.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ifh.h @@ -8,6 +8,7 @@ */ #define IFH_LEN 7 +#define IFH_LEN_BYTES (IFH_LEN * sizeof(u32)) /* Timestamp for frame */ #define IFH_POS_TIMESTAMP 192 diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index 20ee5b28f70a..42be5d0f1f01 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -468,6 +468,7 @@ static const struct net_device_ops lan966x_port_netdev_ops = { .ndo_get_port_parent_id = lan966x_port_get_parent_id, .ndo_eth_ioctl = lan966x_port_ioctl, .ndo_setup_tc = lan966x_tc_setup, + .ndo_bpf = lan966x_xdp, }; bool lan966x_netdevice_check(const struct net_device *dev) @@ -694,6 +695,7 @@ static void lan966x_cleanup_ports(struct lan966x *lan966x) if (port->dev) unregister_netdev(port->dev); + lan966x_xdp_port_deinit(port); if (lan966x->fdma && lan966x->fdma_ndev == port->dev) lan966x_fdma_netdev_deinit(lan966x, port->dev); @@ -760,7 +762,7 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_TC; dev->hw_features |= NETIF_F_HW_TC; - dev->needed_headroom = IFH_LEN * sizeof(u32); + dev->needed_headroom = IFH_LEN_BYTES; eth_hw_addr_gen(dev, lan966x->base_mac, p + 1); @@ -1136,6 +1138,9 @@ static int lan966x_probe(struct platform_device *pdev) lan966x->ports[p]->serdes = serdes; lan966x_port_init(lan966x->ports[p]); + err = lan966x_xdp_port_init(lan966x->ports[p]); + if (err) + goto cleanup_ports; } lan966x_mdb_init(lan966x); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h index 4ec33999e4df..bc93051aa079 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h @@ -9,6 +9,7 @@ #include <linux/phy.h> #include <linux/phylink.h> #include <linux/ptp_clock_kernel.h> +#include <net/page_pool.h> #include <net/pkt_cls.h> #include <net/pkt_sched.h> #include <net/switchdev.h> @@ -100,6 +101,17 @@ enum macaccess_entry_type { ENTRYTYPE_MACV6, }; +/* FDMA return action codes for checking if the frame is valid + * FDMA_PASS, frame is valid and can be used + * FDMA_ERROR, something went wrong, stop getting more frames + * FDMA_DROP, frame is dropped, but continue to get more frames + */ +enum lan966x_fdma_action { + FDMA_PASS = 0, + FDMA_ERROR, + FDMA_DROP, +}; + struct lan966x_port; struct lan966x_db { @@ -150,7 +162,14 @@ struct lan966x_rx { */ u8 page_order; + /* Represents the max size frame that it can receive to the CPU. This + * includes the IFH + VLAN tags + frame + skb_shared_info + */ + u32 max_mtu; + u8 channel_id; + + struct page_pool *page_pool; }; struct lan966x_tx_dcb_buf { @@ -320,6 +339,9 @@ struct lan966x_port { enum netdev_lag_hash hash_type; struct lan966x_port_tc tc; + + struct bpf_prog *xdp_prog; + struct xdp_rxq_info xdp_rxq; }; extern const struct phylink_mac_ops lan966x_phylink_mac_ops; @@ -527,6 +549,17 @@ void lan966x_mirror_port_stats(struct lan966x_port *port, struct flow_stats *stats, bool ingress); +int lan966x_xdp_port_init(struct lan966x_port *port); +void lan966x_xdp_port_deinit(struct lan966x_port *port); +int lan966x_xdp(struct net_device *dev, struct netdev_bpf *xdp); +int lan966x_xdp_run(struct lan966x_port *port, + struct page *page, + u32 data_len); +static inline bool lan966x_xdp_port_present(struct lan966x_port *port) +{ + return !!port->xdp_prog; +} + static inline void __iomem *lan_addr(void __iomem *base[], int id, int tinst, int tcnt, int gbase, int ginst, diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c new file mode 100644 index 000000000000..e77d9f2aad2b --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include <linux/bpf.h> +#include <linux/bpf_trace.h> +#include <linux/filter.h> + +#include "lan966x_main.h" + +static int lan966x_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + struct bpf_prog *old_prog; + + if (!lan966x->fdma) { + NL_SET_ERR_MSG_MOD(xdp->extack, + "Allow to set xdp only when using fdma"); + return -EOPNOTSUPP; + } + + old_prog = xchg(&port->xdp_prog, xdp->prog); + if (old_prog) + bpf_prog_put(old_prog); + + return 0; +} + +int lan966x_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + switch (xdp->command) { + case XDP_SETUP_PROG: + return lan966x_xdp_setup(dev, xdp); + default: + return -EINVAL; + } +} + +int lan966x_xdp_run(struct lan966x_port *port, struct page *page, u32 data_len) +{ + struct bpf_prog *xdp_prog = port->xdp_prog; + struct lan966x *lan966x = port->lan966x; + struct xdp_buff xdp; + u32 act; + + xdp_init_buff(&xdp, PAGE_SIZE << lan966x->rx.page_order, + &port->xdp_rxq); + xdp_prepare_buff(&xdp, page_address(page), IFH_LEN_BYTES, + data_len - IFH_LEN_BYTES, false); + act = bpf_prog_run_xdp(xdp_prog, &xdp); + switch (act) { + case XDP_PASS: + return FDMA_PASS; + default: + bpf_warn_invalid_xdp_action(port->dev, xdp_prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(port->dev, xdp_prog, act); + fallthrough; + case XDP_DROP: + return FDMA_DROP; + } +} + +int lan966x_xdp_port_init(struct lan966x_port *port) +{ + struct lan966x *lan966x = port->lan966x; + + return xdp_rxq_info_reg(&port->xdp_rxq, port->dev, 0, + lan966x->napi.napi_id); +} + +void lan966x_xdp_port_deinit(struct lan966x_port *port) +{ + if (xdp_rxq_info_is_reg(&port->xdp_rxq)) + xdp_rxq_info_unreg(&port->xdp_rxq); +} |