Implement the XHCI operations for allocating and requesting for a secondary interrupter. The secondary interrupter can allow for events for a particular endpoint to be routed to a separate event ring. The event routing is defined when submitting a transfer descriptor to the USB HW. There is a specific field which denotes which interrupter ring to route the event to when the transfer is completed.
An example use case, such as audio packet offloading can utilize a separate event ring, so that these events can be routed to a different processor within the system. The processor would be able to independently submit transfers and handle its completions without intervention from the main processor.
Signed-off-by: Wesley Cheng quic_wcheng@quicinc.com --- drivers/usb/host/xhci-mem.c | 219 ++++++++++++++++++++++++++++------- drivers/usb/host/xhci-plat.c | 2 + drivers/usb/host/xhci.c | 169 ++++++++++++++++++++++++++- drivers/usb/host/xhci.h | 15 +++ 4 files changed, 363 insertions(+), 42 deletions(-)
diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index 81ca2bc1f0be..d5cb4b82ad3d 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -1835,6 +1835,7 @@ void xhci_free_erst(struct xhci_hcd *xhci, struct xhci_erst *erst) void xhci_mem_cleanup(struct xhci_hcd *xhci) { struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + struct xhci_sec *sec, *tmp; int i, j, num_ports;
cancel_delayed_work_sync(&xhci->cmd_timer); @@ -1846,6 +1847,16 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci) xhci->event_ring = NULL; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring");
+ list_for_each_entry_safe(sec, tmp, &xhci->xhci_sec_list, list) { + list_del(&sec->list); + if (sec->event_ring) { + xhci_ring_free(xhci, sec->event_ring); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Freed secondary ring %d", sec->intr_num); + } + kfree(sec); + } + if (xhci->cmd_ring) xhci_ring_free(xhci, xhci->cmd_ring); xhci->cmd_ring = NULL; @@ -2087,18 +2098,18 @@ static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci) return 0; }
-static void xhci_set_hc_event_deq(struct xhci_hcd *xhci) +static void xhci_set_hc_event_deq(struct xhci_hcd *xhci, struct xhci_ring *er, + struct xhci_intr_reg __iomem *ir_set) { u64 temp; dma_addr_t deq;
- deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, - xhci->event_ring->dequeue); + deq = xhci_trb_virt_to_dma(er->deq_seg, er->dequeue); if (!deq) xhci_warn(xhci, "WARN something wrong with SW event ring " "dequeue ptr.\n"); /* Update HC event ring dequeue pointer */ - temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); + temp = xhci_read_64(xhci, &ir_set->erst_dequeue); temp &= ERST_PTR_MASK; /* Don't clear the EHB bit (which is RW1C) because * there might be more events to service. @@ -2108,7 +2119,7 @@ static void xhci_set_hc_event_deq(struct xhci_hcd *xhci) "// Write event ring dequeue pointer, " "preserving EHB bit"); xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp, - &xhci->ir_set->erst_dequeue); + &ir_set->erst_dequeue); }
static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, @@ -2375,10 +2386,159 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) return 0; }
+static void xhci_reset_evt_ring_base(struct xhci_hcd *xhci, struct xhci_ring *er, + struct xhci_erst *erst, struct xhci_intr_reg __iomem *ir_set) +{ + u64 val_64; + + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Set ERST entries to point to event ring."); + /* set the segment table base address */ + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Set ERST base address for ir_set 0 = 0x%llx", + (unsigned long long)erst->erst_dma_addr); + val_64 = xhci_read_64(xhci, &ir_set->erst_base); + val_64 &= ERST_PTR_MASK; + pr_err("after clearing ptr = 0x%llx\n", val_64); + val_64 |= (erst->erst_dma_addr & (u64) ~ERST_PTR_MASK); + pr_err("after clearing ptr = 0x%llx\n", val_64); + + xhci_write_64(xhci, val_64, &ir_set->erst_base); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "done."); + /* Set the event ring dequeue address */ + xhci_set_hc_event_deq(xhci, er, ir_set); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Wrote ERST address to ir_set 0."); +} + +void xhci_handle_sec_intr_events(struct xhci_hcd *xhci, struct xhci_ring *ring, + struct xhci_intr_reg __iomem *ir_set, struct xhci_sec *sec) +{ + union xhci_trb *erdp_trb, *current_trb; + struct xhci_segment *seg; + u64 erdp_reg; + u32 iman_reg; + dma_addr_t deq; + unsigned long segment_offset; + struct xhci_erst *erst = &sec->erst; + + /* disable irq, ack pending interrupt and ack all pending events */ + + iman_reg = readl_relaxed(&ir_set->irq_pending); + iman_reg &= ~IMAN_IE; + writel_relaxed(iman_reg, &ir_set->irq_pending); + iman_reg = readl_relaxed(&ir_set->irq_pending); + if (iman_reg & IMAN_IP) + writel_relaxed(iman_reg, &ir_set->irq_pending); + + /* last acked event trb is in erdp reg */ + erdp_reg = xhci_read_64(xhci, &ir_set->erst_dequeue); + deq = (dma_addr_t)(erdp_reg & ~ERST_PTR_MASK); + if (!deq) { + pr_debug("%s: event ring handling not required\n", __func__); + return; + } + + seg = ring->first_seg; + segment_offset = deq - seg->dma; + + /* find out virtual address of the last acked event trb */ + erdp_trb = current_trb = &seg->trbs[0] + + (segment_offset/sizeof(*current_trb)); + + /* read cycle state of the last acked trb to find out CCS */ + ring->cycle_state = le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE; + + while (1) { + /* last trb of the event ring: toggle cycle state */ + if (current_trb == &seg->trbs[TRBS_PER_SEGMENT - 1]) { + ring->cycle_state ^= 1; + current_trb = &seg->trbs[0]; + } else { + current_trb++; + } + + /* cycle state transition */ + if ((le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE) != + ring->cycle_state) + break; + } + + if (erdp_trb != current_trb) { + deq = xhci_trb_virt_to_dma(ring->deq_seg, current_trb); + if (deq == 0) + xhci_warn(xhci, + "WARN invalid SW event ring dequeue ptr.\n"); + /* Update HC event ring dequeue pointer */ + erdp_reg &= ERST_PTR_MASK; + erdp_reg |= ((u64) deq & (u64) ~ERST_PTR_MASK); + } + + /* Clear the event handler busy flag (RW1C); event ring is empty. */ + erdp_reg |= ERST_EHB; + xhci_write_64(xhci, erdp_reg, &ir_set->erst_dequeue); + + xhci_reset_evt_ring_base(xhci, ring, erst, ir_set); +} + +static int xhci_event_ring_setup(struct xhci_hcd *xhci, struct xhci_ring **er, + struct xhci_erst *erst, struct xhci_intr_reg __iomem *ir_set, + unsigned int intr_num, gfp_t flags) +{ + int ret; + unsigned int val; + + /* + * Event ring setup: Allocate a normal ring, but also setup + * the event ring segment table (ERST). Section 4.9.3. + */ + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring"); + *er = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, + 0, flags); + if (!*er) + return -ENOMEM; + + ret = xhci_alloc_erst(xhci, *er, erst, flags); + if (ret) + return ret; + + /* set ERST count with the number of entries in the segment table */ + val = readl(&ir_set->erst_size); + val &= ERST_SIZE_MASK; + val |= ERST_NUM_SEGS; + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Write ERST size = %i to ir_set 0 (some bits preserved)", + val); + writel(val, &ir_set->erst_size); + + xhci_reset_evt_ring_base(xhci, *er, erst, ir_set); + + return 0; +} + +int xhci_mem_sec_init(struct xhci_hcd *xhci, struct xhci_sec *sec) +{ + int ret; + + sec->ir_set = &xhci->run_regs->ir_set[sec->intr_num]; + ret = xhci_event_ring_setup(xhci, &sec->event_ring, + &sec->erst, sec->ir_set, sec->intr_num, GFP_KERNEL); + if (ret) { + xhci_err(xhci, "sec event ring setup failed inter#%d\n", + sec->intr_num); + return ret; + } + sec->xhci = 0; + + return 0; +} + int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) { dma_addr_t dma; struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + struct xhci_sec *sec; unsigned int val, val2; u64 val_64; u32 page_size, temp; @@ -2497,46 +2657,23 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) /* Set ir_set to interrupt register set 0 */ xhci->ir_set = &xhci->run_regs->ir_set[0];
- /* - * Event ring setup: Allocate a normal ring, but also setup - * the event ring segment table (ERST). Section 4.9.3. - */ - xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring"); - xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, - 0, flags); - if (!xhci->event_ring) - goto fail; - if (xhci_check_trb_in_td_math(xhci) < 0) + ret = xhci_event_ring_setup(xhci, &xhci->event_ring, &xhci->erst, xhci->ir_set, 0, flags); + if (ret < 0) goto fail;
- ret = xhci_alloc_erst(xhci, xhci->event_ring, &xhci->erst, flags); - if (ret) + if (xhci_check_trb_in_td_math(xhci) < 0) goto fail;
- /* set ERST count with the number of entries in the segment table */ - val = readl(&xhci->ir_set->erst_size); - val &= ERST_SIZE_MASK; - val |= ERST_NUM_SEGS; - xhci_dbg_trace(xhci, trace_xhci_dbg_init, - "// Write ERST size = %i to ir_set 0 (some bits preserved)", - val); - writel(val, &xhci->ir_set->erst_size); - - xhci_dbg_trace(xhci, trace_xhci_dbg_init, - "// Set ERST entries to point to event ring."); - /* set the segment table base address */ - xhci_dbg_trace(xhci, trace_xhci_dbg_init, - "// Set ERST base address for ir_set 0 = 0x%llx", - (unsigned long long)xhci->erst.erst_dma_addr); - val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base); - val_64 &= ERST_PTR_MASK; - val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK); - xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base); - - /* Set the event ring dequeue address */ - xhci_set_hc_event_deq(xhci); - xhci_dbg_trace(xhci, trace_xhci_dbg_init, - "Wrote ERST address to ir_set 0."); + /* Setup secondary interrupters (if any) */ + INIT_LIST_HEAD(&xhci->xhci_sec_list); + for (i = 0; i < xhci->max_interrupters; i++) { + sec = kzalloc(sizeof(struct xhci_sec), GFP_KERNEL); + if (sec) { + sec->intr_num = i + 1; + xhci_mem_sec_init(xhci, sec); + list_add_tail(&sec->list, &xhci->xhci_sec_list); + } + }
xhci->isoc_bei_interval = AVOID_BEI_INTERVAL_MAX;
diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 5fb55bf19493..a1b6c17ecf74 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -303,6 +303,8 @@ static int xhci_plat_probe(struct platform_device *pdev)
device_property_read_u32(tmpdev, "imod-interval-ns", &xhci->imod_interval); + device_property_read_u8(tmpdev, "num-hc-interrupters", + &xhci->max_interrupters); }
hcd->usb_phy = devm_usb_get_phy_by_phandle(sysdev, "usb-phy", 0); diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 79d7931c048a..353b06df2000 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -2101,6 +2101,64 @@ int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev, } EXPORT_SYMBOL_GPL(xhci_add_endpoint);
+static int xhci_stop_endpoint(struct usb_hcd *hcd, struct usb_device *udev, + struct usb_host_endpoint *ep) +{ + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + unsigned int ep_index; + struct xhci_virt_device *virt_dev; + struct xhci_command *cmd; + unsigned long flags; + int ret = 0; + + ret = xhci_check_args(hcd, udev, ep, 1, true, __func__); + if (ret <= 0) + return ret; + + cmd = xhci_alloc_command(xhci, true, GFP_NOIO); + if (!cmd) + return -ENOMEM; + + spin_lock_irqsave(&xhci->lock, flags); + virt_dev = xhci->devs[udev->slot_id]; + if (!virt_dev) { + ret = -ENODEV; + goto err; + } + + ep_index = xhci_get_endpoint_index(&ep->desc); + if (virt_dev->eps[ep_index].ring && + virt_dev->eps[ep_index].ring->dequeue) { + ret = xhci_queue_stop_endpoint(xhci, cmd, udev->slot_id, + ep_index, 0); + if (ret) + goto err; + + xhci_ring_cmd_db(xhci); + spin_unlock_irqrestore(&xhci->lock, flags); + + /* Wait for stop endpoint command to finish */ + wait_for_completion(cmd->completion); + + if (cmd->status == COMP_COMMAND_ABORTED || + cmd->status == COMP_STOPPED) { + xhci_warn(xhci, + "stop endpoint command timeout for ep%d%s\n", + usb_endpoint_num(&ep->desc), + usb_endpoint_dir_in(&ep->desc) ? "in" : "out"); + ret = -ETIME; + } + goto free_cmd; + } + +err: + spin_unlock_irqrestore(&xhci->lock, flags); +free_cmd: + xhci_free_command(xhci, cmd); + + return ret; +} + static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev) { struct xhci_input_control_ctx *ctrl_ctx; @@ -5278,6 +5336,110 @@ static void xhci_hcd_init_usb3_data(struct xhci_hcd *xhci, struct usb_hcd *hcd) xhci->usb3_rhub.hcd = hcd; }
+/* + * Free a XHCI interrupter that was previously allocated. Ensure that the + * event ring which was used is reset to the proper state, and recycle the + * interrupter for next use by clearing the XHCI reference. + */ +static int xhci_free_interrupter(struct usb_hcd *hcd, int intr_num) +{ + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + struct xhci_sec *sec; + + mutex_lock(&xhci->mutex); + list_for_each_entry(sec, &xhci->xhci_sec_list, list) { + if (sec->intr_num == intr_num) { + xhci_handle_sec_intr_events(xhci, sec->event_ring, sec->ir_set, sec); + sec->xhci = 0; + } + } + mutex_unlock(&xhci->mutex); + + return 0; +} + +/* + * Reserve a XHCI interrupter, and pass the base address of the event ring for + * this pariticular interrupter back to the client. + */ +static phys_addr_t xhci_update_interrupter(struct usb_hcd *hcd, int intr_num, + dma_addr_t *dma) +{ + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + struct device *dev = hcd->self.sysdev; + struct sg_table sgt; + phys_addr_t pa; + struct xhci_sec *sec; + + if (!HCD_RH_RUNNING(hcd) || + (xhci->xhc_state & XHCI_STATE_HALTED)) + return 0; + + mutex_lock(&xhci->mutex); + list_for_each_entry(sec, &xhci->xhci_sec_list, list) { + if (sec->intr_num == intr_num) { + dma_get_sgtable(dev, &sgt, sec->event_ring->first_seg->trbs, + sec->event_ring->first_seg->dma, TRB_SEGMENT_SIZE); + + *dma = sec->event_ring->first_seg->dma; + + pa = page_to_phys(sg_page(sgt.sgl)); + sg_free_table(&sgt); + sec->xhci = xhci; + mutex_unlock(&xhci->mutex); + + return pa; + } + } + mutex_unlock(&xhci->mutex); + + return 0; +} + +/* Retrieve the transfer ring base address for a specific endpoint. */ +static phys_addr_t xhci_get_xfer_resource(struct usb_device *udev, + struct usb_host_endpoint *ep, dma_addr_t *dma) +{ + struct usb_hcd *hcd = bus_to_hcd(udev->bus); + struct device *dev = hcd->self.sysdev; + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + struct sg_table sgt; + phys_addr_t pa; + int ret; + unsigned int ep_index; + struct xhci_virt_device *virt_dev; + + if (!HCD_RH_RUNNING(hcd)) + return 0; + + ret = xhci_check_args(hcd, udev, ep, 1, true, __func__); + if (ret <= 0) { + xhci_err(xhci, "%s: invalid args\n", __func__); + return 0; + } + + virt_dev = xhci->devs[udev->slot_id]; + ep_index = xhci_get_endpoint_index(&ep->desc); + + if (virt_dev->eps[ep_index].ring && + virt_dev->eps[ep_index].ring->first_seg) { + + dma_get_sgtable(dev, &sgt, + virt_dev->eps[ep_index].ring->first_seg->trbs, + virt_dev->eps[ep_index].ring->first_seg->dma, + TRB_SEGMENT_SIZE); + + *dma = virt_dev->eps[ep_index].ring->first_seg->dma; + + pa = page_to_phys(sg_page(sgt.sgl)); + sg_free_table(&sgt); + + return pa; + } + + return 0; +} + int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) { struct xhci_hcd *xhci; @@ -5321,6 +5483,8 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
xhci->quirks |= quirks; + xhci->max_interrupters = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1), + xhci->max_interrupters);
get_quirks(dev, xhci);
@@ -5454,7 +5618,10 @@ static const struct hc_driver xhci_hc_driver = { .enable_device = xhci_enable_device, .update_hub_device = xhci_update_hub_device, .reset_device = xhci_discover_or_reset_device, - + .update_interrupter = xhci_update_interrupter, + .free_interrupter = xhci_free_interrupter, + .get_transfer_resource = xhci_get_xfer_resource, + .stop_endpoint = xhci_stop_endpoint, /* * scheduling support */ diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index c9f06c5e4e9d..2e694686c849 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1926,6 +1926,7 @@ struct xhci_hcd { struct dentry *debugfs_root; struct dentry *debugfs_slots; struct list_head regset_list; + struct list_head xhci_sec_list;
void *dbc; /* platform-specific data -- must come last */ @@ -1945,6 +1946,17 @@ struct xhci_driver_overrides { void (*reset_bandwidth)(struct usb_hcd *, struct usb_device *); };
+struct xhci_sec { + struct xhci_ring *event_ring; + struct xhci_erst erst; + /* secondary interrupter */ + struct xhci_intr_reg __iomem *ir_set; + struct xhci_hcd *xhci; + int intr_num; + + struct list_head list; +}; + #define XHCI_CFC_DELAY 10
/* convert between an HCD pointer and the corresponding EHCI_HCD */ @@ -2034,6 +2046,9 @@ void xhci_dbg_trace(struct xhci_hcd *xhci, void (*trace)(struct va_format *), /* xHCI memory management */ void xhci_mem_cleanup(struct xhci_hcd *xhci); int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags); +void xhci_handle_sec_intr_events(struct xhci_hcd *xhci, + struct xhci_ring *ring, struct xhci_intr_reg __iomem *ir_set, struct xhci_sec *sec); +int xhci_mem_sec_init(struct xhci_hcd *xhci, struct xhci_sec *sec); void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id); int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, struct usb_device *udev, gfp_t flags); int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev);