diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c index 356fc72840c1..07a0b0606a8f 100644 --- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -55,20 +55,33 @@ struct reset_bulk_data { }; #define PCIE_DMA_OFFSET 0x380000 + #define PCIE_DMA_WR_ENB 0xc -#define PCIE_DMA_CTRL_LO 0x200 -#define PCIE_DMA_CTRL_HI 0x204 -#define PCIE_DMA_XFERSIZE 0x208 -#define PCIE_DMA_SAR_PTR_LO 0x20c -#define PCIE_DMA_SAR_PTR_HI 0x210 -#define PCIE_DMA_DAR_PTR_LO 0x214 -#define PCIE_DMA_DAR_PTR_HI 0x218 +#define PCIE_DMA_WR_CTRL_LO 0x200 +#define PCIE_DMA_WR_CTRL_HI 0x204 +#define PCIE_DMA_WR_XFERSIZE 0x208 +#define PCIE_DMA_WR_SAR_PTR_LO 0x20c +#define PCIE_DMA_WR_SAR_PTR_HI 0x210 +#define PCIE_DMA_WR_DAR_PTR_LO 0x214 +#define PCIE_DMA_WR_DAR_PTR_HI 0x218 #define PCIE_DMA_WR_WEILO 0x18 #define PCIE_DMA_WR_WEIHI 0x1c #define PCIE_DMA_WR_DOORBELL 0x10 #define PCIE_DMA_WR_INT_STATUS 0x4c #define PCIE_DMA_WR_INT_MASK 0x54 #define PCIE_DMA_WR_INT_CLEAR 0x58 + +#define PCIE_DMA_RD_ENB 0x2c +#define PCIE_DMA_RD_CTRL_LO 0x300 +#define PCIE_DMA_RD_CTRL_HI 0x304 +#define PCIE_DMA_RD_XFERSIZE 0x308 +#define PCIE_DMA_RD_SAR_PTR_LO 0x30c +#define PCIE_DMA_RD_SAR_PTR_HI 0x310 +#define PCIE_DMA_RD_DAR_PTR_LO 0x314 +#define PCIE_DMA_RD_DAR_PTR_HI 0x318 +#define PCIE_DMA_RD_WEILO 0x38 +#define PCIE_DMA_RD_WEIHI 0x3c +#define PCIE_DMA_RD_DOORBELL 0x30 #define PCIE_DMA_RD_INT_STATUS 0xa0 #define PCIE_DMA_RD_INT_MASK 0xa8 #define PCIE_DMA_RD_INT_CLEAR 0xac @@ -1193,64 +1206,119 @@ static int rk_pcie_reset_grant_ctrl(struct rk_pcie *rk_pcie, return ret; } -static void rk_pcie_start_dma_dwc(struct dma_trx_obj *obj) +static void rk_pcie_start_dma_rd(struct dma_trx_obj *obj, int ctr_off) { struct rk_pcie *rk_pcie = dev_get_drvdata(obj->dev); + struct dma_table *cur = obj->cur; + + dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_ENB, + cur->enb.asdword); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_RD_CTRL_LO, + cur->ctx_reg.ctrllo.asdword); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_RD_CTRL_HI, + cur->ctx_reg.ctrlhi.asdword); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_RD_XFERSIZE, + cur->ctx_reg.xfersize); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_RD_SAR_PTR_LO, + cur->ctx_reg.sarptrlo); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_RD_SAR_PTR_HI, + cur->ctx_reg.sarptrhi); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_RD_DAR_PTR_LO, + cur->ctx_reg.darptrlo); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_RD_DAR_PTR_HI, + cur->ctx_reg.darptrhi); + dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_DOORBELL, + cur->start.asdword); +} + +static void rk_pcie_start_dma_wr(struct dma_trx_obj *obj, int ctr_off) +{ + struct rk_pcie *rk_pcie = dev_get_drvdata(obj->dev); + struct dma_table *cur = obj->cur; dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_ENB, - obj->cur->wr_enb.asdword); - dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_CTRL_LO, - obj->cur->ctx_reg.ctrllo.asdword); - dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_CTRL_HI, - obj->cur->ctx_reg.ctrlhi.asdword); - dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_XFERSIZE, - obj->cur->ctx_reg.xfersize); - dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_SAR_PTR_LO, - obj->cur->ctx_reg.sarptrlo); - dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_SAR_PTR_HI, - obj->cur->ctx_reg.sarptrhi); - dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_DAR_PTR_LO, - obj->cur->ctx_reg.darptrlo); - dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_DAR_PTR_HI, - obj->cur->ctx_reg.darptrhi); - dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_WEILO, - obj->cur->wr_weilo.asdword); + cur->enb.asdword); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_WR_CTRL_LO, + cur->ctx_reg.ctrllo.asdword); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_WR_CTRL_HI, + cur->ctx_reg.ctrlhi.asdword); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_WR_XFERSIZE, + cur->ctx_reg.xfersize); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_WR_SAR_PTR_LO, + cur->ctx_reg.sarptrlo); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_WR_SAR_PTR_HI, + cur->ctx_reg.sarptrhi); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_WR_DAR_PTR_LO, + cur->ctx_reg.darptrlo); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_WR_DAR_PTR_HI, + cur->ctx_reg.darptrhi); + dw_pcie_writel_dbi(rk_pcie->pci, ctr_off + PCIE_DMA_WR_WEILO, + cur->weilo.asdword); dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_DOORBELL, - obj->cur->start.asdword); + cur->start.asdword); +} + +static void rk_pcie_start_dma_dwc(struct dma_trx_obj *obj) +{ + int dir = obj->cur->dir; + int chn = obj->cur->chn; + + int ctr_off = PCIE_DMA_OFFSET + chn * 0x200; + + if (dir == DMA_FROM_BUS) + rk_pcie_start_dma_rd(obj, ctr_off); + else if (dir == DMA_TO_BUS) + rk_pcie_start_dma_wr(obj, ctr_off); } static void rk_pcie_config_dma_dwc(struct dma_table *table) { - table->wr_enb.enb = 0x1; + table->enb.enb = 0x1; table->ctx_reg.ctrllo.lie = 0x1; table->ctx_reg.ctrllo.rie = 0x0; table->ctx_reg.ctrllo.td = 0x1; table->ctx_reg.ctrlhi.asdword = 0x0; table->ctx_reg.xfersize = table->buf_size; - table->ctx_reg.sarptrlo = (u32)(table->local & 0xffffffff); - table->ctx_reg.sarptrhi = (u32)(table->local >> 32); - table->ctx_reg.darptrlo = (u32)(table->bus & 0xffffffff); - table->ctx_reg.darptrhi = (u32)(table->bus >> 32); - table->wr_weilo.weight0 = 0x0; + if (table->dir == DMA_FROM_BUS) { + table->ctx_reg.sarptrlo = (u32)(table->bus & 0xffffffff); + table->ctx_reg.sarptrhi = (u32)(table->bus >> 32); + table->ctx_reg.darptrlo = (u32)(table->local & 0xffffffff); + table->ctx_reg.darptrhi = (u32)(table->local >> 32); + } else if (table->dir == DMA_TO_BUS) { + table->ctx_reg.sarptrlo = (u32)(table->local & 0xffffffff); + table->ctx_reg.sarptrhi = (u32)(table->local >> 32); + table->ctx_reg.darptrlo = (u32)(table->bus & 0xffffffff); + table->ctx_reg.darptrhi = (u32)(table->bus >> 32); + } + table->weilo.weight0 = 0x0; table->start.stop = 0x0; - table->start.chnl = PCIE_DMA_CHN0; + table->start.chnl = table->chn; } static inline void rk_pcie_handle_dma_interrupt(struct rk_pcie *rk_pcie) { struct dma_trx_obj *obj = rk_pcie->dma_obj; + struct dma_table *cur; if (!obj) return; + cur = obj->cur; + if (!cur) { + pr_err("no pcie dma table\n"); + return; + } + obj->dma_free = true; obj->irq_num++; - if (list_empty(&obj->tbl_list)) { - if (obj->dma_free && - obj->loop_count >= obj->loop_count_threshold) - complete(&obj->done); + if (cur->dir == DMA_TO_BUS) { + if (list_empty(&obj->tbl_list)) { + if (obj->dma_free && + obj->loop_count >= obj->loop_count_threshold) + complete(&obj->done); + } } } @@ -1268,20 +1336,37 @@ static irqreturn_t rk_pcie_sys_irq_handler(int irq, void *arg) if (rk_pcie->dma_obj && rk_pcie->dma_obj->cur) chn = rk_pcie->dma_obj->cur->chn; - if (status.donesta & BIT(0)) { + if (status.donesta & BIT(chn)) { clears.doneclr = 0x1 << chn; dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_CLEAR, clears.asdword); rk_pcie_handle_dma_interrupt(rk_pcie); } - if (status.abortsta & BIT(0)) { + if (status.abortsta & BIT(chn)) { dev_err(rk_pcie->pci->dev, "%s, abort\n", __func__); clears.abortclr = 0x1 << chn; dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_CLEAR, clears.asdword); } + status.asdword = dw_pcie_readl_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + + PCIE_DMA_RD_INT_STATUS); + + if (status.donesta & BIT(chn)) { + clears.doneclr = 0x1 << chn; + dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + + PCIE_DMA_RD_INT_CLEAR, clears.asdword); + rk_pcie_handle_dma_interrupt(rk_pcie); + } + + if (status.abortsta & BIT(chn)) { + dev_err(rk_pcie->pci->dev, "%s, abort\n", __func__); + clears.abortclr = 0x1 << chn; + dw_pcie_writel_dbi(rk_pcie->pci, PCIE_DMA_OFFSET + + PCIE_DMA_RD_INT_CLEAR, clears.asdword); + } + reg = rk_pcie_readl_apb(rk_pcie, PCIE_CLIENT_INTR_STATUS_MISC); if (reg & BIT(2)) { /* Setup command register */ diff --git a/drivers/pci/controller/rockchip-pcie-dma.c b/drivers/pci/controller/rockchip-pcie-dma.c index 39ce402d724e..7f9a369d4b0c 100644 --- a/drivers/pci/controller/rockchip-pcie-dma.c +++ b/drivers/pci/controller/rockchip-pcie-dma.c @@ -113,16 +113,7 @@ #define NODE_SIZE (sizeof(unsigned int)) #define PCIE_DMA_ACK_BLOCK_SIZE (NODE_SIZE * 8) -#define PCIE_DMA_BUF_SIZE SZ_128K -#define PCIE_DMA_BUF_CNT 8 -#define PCIE_DMA_RD_BUF_SIZE (PCIE_DMA_BUF_SIZE * PCIE_DMA_BUF_CNT) -#define PCIE_DMA_WR_BUF_SIZE (PCIE_DMA_BUF_SIZE * PCIE_DMA_BUF_CNT) -#define PCIE_DMA_ACK_BASE (PCIE_DMA_RD_BUF_SIZE + PCIE_DMA_WR_BUF_SIZE) - -#define PCIE_DMA_SET_DATA_CHECK_POS (PCIE_DMA_BUF_SIZE - 0x4) -#define PCIE_DMA_SET_LOCAL_IDX_POS (PCIE_DMA_BUF_SIZE - 0x8) -#define PCIE_DMA_SET_BUF_SIZE_POS (PCIE_DMA_BUF_SIZE - 0xc) -#define PCIE_DMA_SET_CHK_SUM_POS (PCIE_DMA_BUF_SIZE - 0x10) +#define PCIE_DMA_BUF_CNT 8 #define PCIE_DMA_DATA_CHECK 0x12345678 #define PCIE_DMA_DATA_ACK_CHECK 0xdeadbeef @@ -132,11 +123,12 @@ #define PCIE_DMA_CHN0 0x0 static int enable_check_sum; - struct pcie_misc_dev { struct miscdevice dev; struct dma_trx_obj *obj; }; +static void *rk_pcie_map_kernel(phys_addr_t start, size_t len); +static void rk_pcie_unmap_kernel(void *vaddr); static inline bool is_rc(struct dma_trx_obj *obj) { @@ -158,7 +150,7 @@ static unsigned int rk_pcie_check_sum(unsigned int *src, int size) static void rk_pcie_prepare_dma(struct dma_trx_obj *obj, unsigned int idx, unsigned int bus_idx, unsigned int local_idx, size_t buf_size, - enum transfer_type type) + enum transfer_type type, int chn) { struct device *dev = obj->dev; phys_addr_t local, bus; @@ -171,58 +163,80 @@ static void rk_pcie_prepare_dma(struct dma_trx_obj *obj, case PCIE_DMA_DATA_SND: table = obj->table[PCIE_DMA_DATA_SND_TABLE_OFFSET + local_idx]; table->type = PCIE_DMA_DATA_SND; - local = obj->mem_start + local_idx * PCIE_DMA_BUF_SIZE; - virt = obj->mem_base + local_idx * PCIE_DMA_BUF_SIZE; - bus = obj->mem_start + bus_idx * PCIE_DMA_BUF_SIZE; + table->dir = DMA_TO_BUS; + local = obj->local_mem_start + local_idx * obj->buffer_size; + bus = obj->remote_mem_start + bus_idx * obj->buffer_size; + virt = obj->local_mem_base + local_idx * obj->buffer_size; if (!is_rc(obj)) { - local += PCIE_DMA_RD_BUF_SIZE; - virt += PCIE_DMA_RD_BUF_SIZE; - bus += PCIE_DMA_WR_BUF_SIZE; + local += obj->rd_buf_size; + virt += obj->rd_buf_size; + bus += obj->wr_buf_size; } + obj->begin = ktime_get(); dma_sync_single_for_device(dev, local, buf_size, DMA_TO_DEVICE); + obj->end = ktime_get(); - writel(PCIE_DMA_DATA_CHECK, virt + PCIE_DMA_SET_DATA_CHECK_POS); - writel(local_idx, virt + PCIE_DMA_SET_LOCAL_IDX_POS); - writel(buf_size, virt + PCIE_DMA_SET_BUF_SIZE_POS); + obj->cache_time_total += ktime_to_ns(ktime_sub(obj->end, obj->begin)); + + writel(PCIE_DMA_DATA_CHECK, virt + obj->set_data_check_pos); + writel(local_idx, virt + obj->set_local_idx_pos); + writel(buf_size, virt + obj->set_buf_size_pos); if (enable_check_sum) { checksum = rk_pcie_check_sum(virt, SZ_1M - 0x10); - writel(checksum, virt + PCIE_DMA_SET_CHK_SUM_POS); + writel(checksum, virt + obj->set_chk_sum_pos); } - buf_size = PCIE_DMA_BUF_SIZE; + buf_size = obj->buffer_size; break; case PCIE_DMA_DATA_RCV_ACK: table = obj->table[PCIE_DMA_DATA_RCV_ACK_TABLE_OFFSET + idx]; table->type = PCIE_DMA_DATA_RCV_ACK; - local = obj->mem_start + PCIE_DMA_ACK_BASE + idx * NODE_SIZE; - virt = obj->mem_base + PCIE_DMA_ACK_BASE + idx * NODE_SIZE; + table->dir = DMA_TO_BUS; + local = obj->local_mem_start + obj->ack_base + idx * NODE_SIZE; + virt = obj->local_mem_base + obj->ack_base + idx * NODE_SIZE; + bus = obj->remote_mem_start + obj->ack_base + idx * NODE_SIZE; if (is_rc(obj)) { local += PCIE_DMA_ACK_BLOCK_SIZE; + bus += PCIE_DMA_ACK_BLOCK_SIZE; virt += PCIE_DMA_ACK_BLOCK_SIZE; } - bus = local; writel(PCIE_DMA_DATA_ACK_CHECK, virt); break; case PCIE_DMA_DATA_FREE_ACK: table = obj->table[PCIE_DMA_DATA_FREE_ACK_TABLE_OFFSET + idx]; table->type = PCIE_DMA_DATA_FREE_ACK; - local = obj->mem_start + PCIE_DMA_ACK_BASE + idx * NODE_SIZE; - virt = obj->mem_base + PCIE_DMA_ACK_BASE + idx * NODE_SIZE; + table->dir = DMA_TO_BUS; + local = obj->local_mem_start + obj->ack_base + idx * NODE_SIZE; + bus = obj->remote_mem_start + obj->ack_base + idx * NODE_SIZE; + virt = obj->local_mem_base + obj->ack_base + idx * NODE_SIZE; if (is_rc(obj)) { local += 3 * PCIE_DMA_ACK_BLOCK_SIZE; + bus += 3 * PCIE_DMA_ACK_BLOCK_SIZE; virt += 3 * PCIE_DMA_ACK_BLOCK_SIZE; } else { local += 2 * PCIE_DMA_ACK_BLOCK_SIZE; + bus += 2 * PCIE_DMA_ACK_BLOCK_SIZE; virt += 2 * PCIE_DMA_ACK_BLOCK_SIZE; } - bus = local; writel(PCIE_DMA_DATA_FREE_ACK_CHECK, virt); break; + case PCIE_DMA_READ_REMOTE: + table = obj->table[PCIE_DMA_DATA_READ_REMOTE_TABLE_OFFSET + local_idx]; + table->type = PCIE_DMA_READ_REMOTE; + table->dir = DMA_FROM_BUS; + local = obj->local_mem_start + local_idx * obj->buffer_size; + bus = obj->remote_mem_start + bus_idx * obj->buffer_size; + if (!is_rc(obj)) { + local += obj->rd_buf_size; + bus += obj->wr_buf_size; + } + buf_size = obj->buffer_size; + break; default: dev_err(dev, "type = %d not support\n", type); return; @@ -231,6 +245,7 @@ static void rk_pcie_prepare_dma(struct dma_trx_obj *obj, table->buf_size = buf_size; table->bus = bus; table->local = local; + table->chn = chn; if (!obj->config_dma_func) { WARN_ON(1); @@ -268,15 +283,6 @@ static void rk_pcie_dma_trx_work(struct work_struct *work) } } -static int rk_pcie_scan_thread(void *data) -{ - struct dma_trx_obj *obj = (struct dma_trx_obj *)data; - - hrtimer_start(&obj->scan_timer, - ktime_set(0, 500 * 1000 * 1000), HRTIMER_MODE_REL); - return 0; -} - static void rk_pcie_clear_ack(void *addr) { writel(0x0, addr); @@ -298,22 +304,30 @@ static enum hrtimer_restart rk_pcie_scan_timer(struct hrtimer *timer) struct dma_trx_obj, scan_timer); unsigned int check_sum, check_sum_tmp; + if (!obj->remote_mem_start) { + if (is_rc(obj)) + obj->remote_mem_start = readl(obj->region_base + 0x4); + else + obj->remote_mem_start = readl(obj->region_base); + goto continue_scan; + } + for (i = 0; i < PCIE_DMA_BUF_CNT; i++) { - sda_base = obj->mem_base + PCIE_DMA_BUF_SIZE * i; + sda_base = obj->local_mem_base + obj->buffer_size * i; if (is_rc(obj)) - scan_data_addr = sda_base + PCIE_DMA_WR_BUF_SIZE; + scan_data_addr = sda_base + obj->rd_buf_size; else scan_data_addr = sda_base; - sdv = readl(scan_data_addr + PCIE_DMA_SET_DATA_CHECK_POS); - idx = readl(scan_data_addr + PCIE_DMA_SET_LOCAL_IDX_POS); + sdv = readl(scan_data_addr + obj->set_data_check_pos); + idx = readl(scan_data_addr + obj->set_local_idx_pos); if (sdv == PCIE_DMA_DATA_CHECK) { if (!need_ack) need_ack = true; if (enable_check_sum) { - check_sum = readl(scan_data_addr + PCIE_DMA_SET_CHK_SUM_POS); + check_sum = readl(scan_data_addr + obj->set_chk_sum_pos); check_sum_tmp = rk_pcie_check_sum(scan_data_addr, SZ_1M - 0x10); if (check_sum != check_sum_tmp) { pr_err("checksum[%d] failed, 0x%x, should be 0x%x\n", @@ -321,21 +335,21 @@ static enum hrtimer_restart rk_pcie_scan_timer(struct hrtimer *timer) print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, 32, 4, scan_data_addr, SZ_1M, false); } - writel(0x0, scan_data_addr + PCIE_DMA_SET_CHK_SUM_POS); + writel(0x0, scan_data_addr + obj->set_chk_sum_pos); } - writel(0x0, scan_data_addr + PCIE_DMA_SET_DATA_CHECK_POS); + writel(0x0, scan_data_addr + obj->set_data_check_pos); set_bit(i, &obj->local_read_available); rk_pcie_prepare_dma(obj, idx, 0, 0, 0x4, - PCIE_DMA_DATA_RCV_ACK); + PCIE_DMA_DATA_RCV_ACK, PCIE_DMA_DEFAULT_CHN); } } if (need_ack || !list_empty(&obj->tbl_list)) queue_work(obj->dma_trx_wq, &obj->dma_trx_work); - scan_ack_addr = obj->mem_base + PCIE_DMA_ACK_BASE; - scan_user_addr = obj->mem_base + PCIE_DMA_ACK_BASE; + scan_ack_addr = obj->local_mem_base + obj->ack_base; + scan_user_addr = obj->local_mem_base + obj->ack_base; if (is_rc(obj)) { scan_user_addr += PCIE_DMA_ACK_BLOCK_SIZE * 2; @@ -366,6 +380,7 @@ static enum hrtimer_restart rk_pcie_scan_timer(struct hrtimer *timer) wake_up(&obj->event_queue); } +continue_scan: hrtimer_add_expires(&obj->scan_timer, ktime_set(0, 100 * 1000)); return HRTIMER_RESTART; @@ -401,8 +416,10 @@ static int rk_pcie_misc_release(struct inode *inode, struct file *filp) struct dma_trx_obj *obj = filp->private_data; mutex_lock(&obj->count_mutex); + if (--obj->ref_count) goto still_opened; + hrtimer_cancel(&obj->scan_timer); pr_info("Close pcie misc device\n"); @@ -419,13 +436,32 @@ static int rk_pcie_misc_mmap(struct file *filp, int err; err = remap_pfn_range(vma, vma->vm_start, - __phys_to_pfn(obj->mem_start), + __phys_to_pfn(obj->local_mem_start), size, vma->vm_page_prot); if (err) return -EAGAIN; return 0; } +static void rk_pcie_send_addr_to_remote(struct dma_trx_obj *obj) +{ + struct dma_table *table; + + /* Temporary use to send local buffer address to remote */ + table = obj->table[PCIE_DMA_DATA_SND_TABLE_OFFSET]; + table->type = PCIE_DMA_DATA_SND; + table->dir = DMA_TO_BUS; + table->buf_size = 0x4; + if (is_rc(obj)) + table->local = obj->region_start; + else + table->local = obj->region_start + 0x4; + table->bus = table->local; + table->chn = PCIE_DMA_DEFAULT_CHN; + obj->config_dma_func(table); + obj->cur = table; + obj->start_dma_func(obj); +} static long rk_pcie_misc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) @@ -438,6 +474,7 @@ static long rk_pcie_misc_ioctl(struct file *filp, unsigned int cmd, void __user *uarg = (void __user *)arg; int ret; int i; + phys_addr_t addr_send_to_remote; if (copy_from_user(&msg, uarg, sizeof(msg)) != 0) { dev_err(dev, "failed to copy argument into kernel space\n"); @@ -452,13 +489,13 @@ static long rk_pcie_misc_ioctl(struct file *filp, unsigned int cmd, break; case PCIE_DMA_GET_LOCAL_READ_BUFFER_INDEX: msg_to_user.lra = obj->local_read_available; - addr = obj->mem_start; + addr = obj->local_mem_start; if (is_rc(obj)) - addr += PCIE_DMA_WR_BUF_SIZE; + addr += obj->rd_buf_size; /* by kernel auto or by user to invalidate cache */ for (i = 0; i < PCIE_DMA_BUF_CNT; i++) { if (test_bit(i, &obj->local_read_available)) - dma_sync_single_for_cpu(dev, addr + i * PCIE_DMA_BUF_SIZE, PCIE_DMA_BUF_SIZE, DMA_FROM_DEVICE); + dma_sync_single_for_cpu(dev, addr + i * obj->buffer_size, obj->buffer_size, DMA_FROM_DEVICE); } ret = copy_to_user(uarg, &msg_to_user, sizeof(msg)); @@ -480,10 +517,10 @@ static long rk_pcie_misc_ioctl(struct file *filp, unsigned int cmd, } break; case PCIE_DMA_SYNC_BUFFER_FOR_CPU: - addr = obj->mem_start + msg.in.idx * PCIE_DMA_BUF_SIZE; + addr = obj->local_mem_start + msg.in.idx * obj->buffer_size; if (is_rc(obj)) - addr += PCIE_DMA_WR_BUF_SIZE; - dma_sync_single_for_cpu(dev, addr, PCIE_DMA_BUF_SIZE, + addr += obj->rd_buf_size; + dma_sync_single_for_cpu(dev, addr, obj->buffer_size, DMA_FROM_DEVICE); break; case PCIE_DMA_WAIT_TRANSFER_COMPLETE: @@ -494,22 +531,85 @@ static long rk_pcie_misc_ioctl(struct file *filp, unsigned int cmd, return ret; } + obj->cache_time_avarage = obj->cache_time_total / obj->loop_count; + + pr_debug("cache_time: total = %lld, average = %lld, count = %d, size = 0x%x\n", + obj->cache_time_total, obj->cache_time_avarage, + obj->loop_count, obj->buffer_size); + + obj->cache_time_avarage = 0; + obj->cache_time_total = 0; + obj->loop_count = 0; break; case PCIE_DMA_SET_LOOP_COUNT: obj->loop_count_threshold = msg.count; pr_info("threshold = %d\n", obj->loop_count_threshold); break; + case PCIE_DMA_GET_TOTAL_BUFFER_SIZE: + msg_to_user.total_buffer_size = obj->local_mem_size; + ret = copy_to_user(uarg, &msg_to_user, sizeof(msg)); + if (ret) { + dev_err(dev, "failed to get write buffer index\n"); + return -EFAULT; + } + break; + case PCIE_DMA_SET_BUFFER_SIZE: + obj->buffer_size = msg.buffer_size; + pr_debug("buffer_size = %d\n", obj->buffer_size); + obj->rd_buf_size = obj->buffer_size * PCIE_DMA_BUF_CNT; + obj->wr_buf_size = obj->buffer_size * PCIE_DMA_BUF_CNT; + obj->ack_base = obj->rd_buf_size + obj->wr_buf_size; + obj->set_data_check_pos = obj->buffer_size - 0x4; + obj->set_local_idx_pos = obj->buffer_size - 0x8; + obj->set_buf_size_pos = obj->buffer_size - 0xc; + obj->set_chk_sum_pos = obj->buffer_size - 0x10; + break; + case PCIE_DMA_READ_FROM_REMOTE: + pr_debug("read buffer from : %d to local : %d\n", + msg.in.r_widx, msg.in.l_widx); + break; + case PCIE_DMA_USER_SET_BUF_ADDR: + /* If msg.local_addr valid, use msg.local_addr for local buffer, + * and should be contiguous physical address. + * If msg.local is zero, local buffer get from DT reserved. + * Anyway local buffer address should send to remote, then + * remote know where to send data to. + * Should finish this case first before send data. + */ + if (msg.local_addr) { + pr_debug("local_addr = %pa\n", &msg.local_addr); + addr_send_to_remote = msg.local_addr; + obj->local_mem_start = msg.local_addr; + /* Unmap previous */ + rk_pcie_unmap_kernel(obj->local_mem_base); + /* Remap userspace's buffer to kernel */ + obj->local_mem_base = rk_pcie_map_kernel(obj->local_mem_start, + obj->buffer_size * PCIE_DMA_BUF_CNT * 2 + SZ_4K); + if (!obj->local_mem_base) + return -EFAULT; + } else { + addr_send_to_remote = obj->local_mem_start; + } + if (is_rc(obj)) + writel(addr_send_to_remote, obj->region_base); + else + writel(addr_send_to_remote, obj->region_base + 0x4); + rk_pcie_send_addr_to_remote(obj); + hrtimer_start(&obj->scan_timer, + ktime_set(0, 1 * 1000 * 1000 * 1000), HRTIMER_MODE_REL); + break; default: pr_info("%s, %d, cmd : %x not support\n", __func__, __LINE__, cmd); return -EFAULT; } - if (cmd == PCIE_DMA_START || + if (cmd == PCIE_DMA_START || cmd == PCIE_DMA_READ_FROM_REMOTE || cmd == PCIE_DMA_SET_LOCAL_READ_BUFFER_INDEX) { rk_pcie_prepare_dma(obj, msg.in.idx, msg.in.r_widx, - msg.in.l_widx, msg.in.size, msg.in.type); + msg.in.l_widx, msg.in.size, msg.in.type, + msg.in.chn); queue_work(obj->dma_trx_wq, &obj->dma_trx_work); } @@ -642,10 +742,7 @@ static int rk_pcie_dma_table_alloc(struct dma_trx_obj *obj) goto free_table; } - if (is_rc(obj)) - table->dir = DMA_TO_BUS; - - table->chn = PCIE_DMA_CHN0; + table->chn = PCIE_DMA_DEFAULT_CHN; INIT_LIST_HEAD(&table->tbl_node); obj->table[i] = table; } @@ -665,6 +762,9 @@ static int rk_pcie_debugfs_trx_show(struct seq_file *s, void *v) struct dma_trx_obj *dma_obj = s->private; bool list = list_empty(&dma_obj->tbl_list); + seq_printf(s, "version = %x,", dma_obj->version); + seq_printf(s, "last:%s,", + dma_obj->cur ? (dma_obj->cur->dir == DMA_FROM_BUS ? "read" : "write") : "no trx"); seq_printf(s, "irq_num = %ld, loop_count = %d,", dma_obj->irq_num, dma_obj->loop_count); seq_printf(s, "loop_threshold = %d,", @@ -742,17 +842,48 @@ struct dma_trx_obj *rk_pcie_dma_obj_probe(struct device *dev) return ERR_PTR(-ENODEV); } - obj->mem_start = reg.start; - obj->mem_size = resource_size(®); - obj->mem_base = rk_pcie_map_kernel(obj->mem_start, obj->mem_size); - - if (!obj->mem_base) + obj->local_mem_start = reg.start; + obj->local_mem_size = resource_size(®); + obj->local_mem_base = rk_pcie_map_kernel(obj->local_mem_start, + obj->local_mem_size); + if (!obj->local_mem_base) return ERR_PTR(-ENOMEM); + mem = of_parse_phandle(np, "memory-region1", 0); + if (!mem) { + dev_err(dev, "missing \"memory-region1\" property\n"); + obj = ERR_PTR(-ENODEV); + goto unmap_local_mem_region; + } + + ret = of_address_to_resource(mem, 0, ®); + if (ret < 0) { + dev_err(dev, "missing \"reg\" property\n"); + obj = ERR_PTR(-ENODEV); + goto unmap_local_mem_region; + } + + obj->region_start = reg.start; + obj->region_size = resource_size(®); + obj->region_base = rk_pcie_map_kernel(obj->region_start, + obj->region_size); + if (!obj->region_base) { + dev_err(dev, "mapping region_base error\n"); + obj = ERR_PTR(-ENOMEM); + goto unmap_local_mem_region; + } + if (!is_rc(obj)) + writel(0x0, obj->region_base); + else + writel(0x0, obj->region_base + 0x4); + ret = rk_pcie_dma_table_alloc(obj); - if (ret) - return ERR_PTR(-ENOMEM); + if (ret) { + dev_err(dev, "rk_pcie_dma_table_alloc error\n"); + obj = ERR_PTR(-ENOMEM); + goto unmap_region; + } obj->dma_trx_wq = create_singlethread_workqueue("dma_trx_wq"); INIT_WORK(&obj->dma_trx_work, rk_pcie_dma_trx_work); @@ -764,17 +895,10 @@ struct dma_trx_obj *rk_pcie_dma_obj_probe(struct device *dev) hrtimer_init_on_stack(&obj->scan_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); obj->scan_timer.function = rk_pcie_scan_timer; - obj->scan_thread = kthread_run(rk_pcie_scan_thread, (void *)obj, - "scan_thread"); - if (!obj->scan_thread) { - dev_err(dev, "kthread_run failed\n"); - obj = ERR_PTR(-EINVAL); - goto free_dma_table; - } - obj->irq_num = 0; obj->loop_count_threshold = 0; obj->ref_count = 0; + obj->version = 0x2; init_completion(&obj->done); mutex_init(&obj->count_mutex); @@ -794,6 +918,11 @@ struct dma_trx_obj *rk_pcie_dma_obj_probe(struct device *dev) return obj; free_dma_table: rk_pcie_dma_table_free(obj, PCIE_DMA_TABLE_NUM); +unmap_region: + rk_pcie_unmap_kernel(obj->region_base); +unmap_local_mem_region: + rk_pcie_unmap_kernel(obj->local_mem_base); + return obj; } EXPORT_SYMBOL_GPL(rk_pcie_dma_obj_probe); @@ -803,7 +932,7 @@ void rk_pcie_dma_obj_remove(struct dma_trx_obj *obj) hrtimer_cancel(&obj->scan_timer); destroy_hrtimer_on_stack(&obj->scan_timer); rk_pcie_delete_misc(obj); - rk_pcie_unmap_kernel(obj->mem_base); + rk_pcie_unmap_kernel(obj->local_mem_base); rk_pcie_dma_table_free(obj, PCIE_DMA_TABLE_NUM); destroy_workqueue(obj->dma_trx_wq); diff --git a/drivers/pci/controller/rockchip-pcie-dma.h b/drivers/pci/controller/rockchip-pcie-dma.h index 50e203b7f51f..a5d923445224 100644 --- a/drivers/pci/controller/rockchip-pcie-dma.h +++ b/drivers/pci/controller/rockchip-pcie-dma.h @@ -2,18 +2,23 @@ /* * Copyright (C) 2018 Rockchip Electronics Co., Ltd. */ +#ifndef __SOC_ROCKCHIP_PCIE_DMA_TRX_H +#define __SOC_ROCKCHIP_PCIE_DMA_TRX_H #include -#define PCIE_DMA_TABLE_NUM 24 +#define PCIE_DMA_TABLE_NUM 32 #define PCIE_DMA_TRX_TYPE_NUM 3 #define PCIE_DMA_CHN0 0x0 +#define PCIE_DMA_CHN1 0x1 +#define PCIE_DMA_DEFAULT_CHN PCIE_DMA_CHN0 #define PCIE_DMA_DATA_SND_TABLE_OFFSET 0x0 #define PCIE_DMA_DATA_RCV_ACK_TABLE_OFFSET 0x8 #define PCIE_DMA_DATA_FREE_ACK_TABLE_OFFSET 0x10 +#define PCIE_DMA_DATA_READ_REMOTE_TABLE_OFFSET 0x18 enum dma_dir { DMA_FROM_BUS, @@ -142,10 +147,10 @@ struct dma_table { u32 dir; u32 type; struct list_head tbl_node; - union enb wr_enb; + union enb enb; struct ctx_regs ctx_reg; - union weight wr_weilo; - union weight wr_weihi; + union weight weilo; + union weight weihi; union db start; phys_addr_t local; phys_addr_t bus; @@ -156,9 +161,13 @@ struct dma_trx_obj { struct device *dev; int loop_count; int loop_count_threshold; - void *mem_base; - phys_addr_t mem_start; - size_t mem_size; + void *local_mem_base; + phys_addr_t local_mem_start; + size_t local_mem_size; + phys_addr_t remote_mem_start; + void *region_base; + phys_addr_t region_start; + size_t region_size; int dma_free; unsigned long local_write_available; unsigned long local_read_available; @@ -170,7 +179,6 @@ struct dma_trx_obj { struct workqueue_struct *dma_trx_wq; struct dma_table *table[PCIE_DMA_TABLE_NUM]; struct dma_table *cur; - struct task_struct *scan_thread; struct hrtimer scan_timer; int busno; void *priv; @@ -182,9 +190,22 @@ struct dma_trx_obj { struct pcie_misc_dev *pcie_dev; void (*start_dma_func)(struct dma_trx_obj *obj); void (*config_dma_func)(struct dma_table *table); + ktime_t begin; + ktime_t end; + u64 cache_time_total; + u64 cache_time_avarage; + u32 buffer_size; + u32 rd_buf_size; + u32 wr_buf_size; + u32 ack_base; + u32 set_data_check_pos; + u32 set_local_idx_pos; + u32 set_buf_size_pos; + u32 set_chk_sum_pos; + u32 version; }; -#ifdef CONFIG_ROCKCHIP_PCIE_DMA_OBJ +#if IS_ENABLED(CONFIG_ROCKCHIP_PCIE_DMA_OBJ) struct dma_trx_obj *rk_pcie_dma_obj_probe(struct device *dev); void rk_pcie_dma_obj_remove(struct dma_trx_obj *obj); #else @@ -197,3 +218,5 @@ static inline void rk_pcie_dma_obj_remove(struct dma_trx_obj *obj) { } #endif + +#endif diff --git a/include/uapi/linux/rk-pcie-dma.h b/include/uapi/linux/rk-pcie-dma.h index 6087419eff2a..a35201d068b1 100644 --- a/include/uapi/linux/rk-pcie-dma.h +++ b/include/uapi/linux/rk-pcie-dma.h @@ -2,11 +2,14 @@ /* * Copyright (C) 2018 Rockchip Electronics Co., Ltd. */ +#ifndef _UAPI__PCIE_DMA_TRX_H__ +#define _UAPI__PCIE_DMA_TRX_H__ enum transfer_type { PCIE_DMA_DATA_SND, PCIE_DMA_DATA_RCV_ACK, PCIE_DMA_DATA_FREE_ACK, + PCIE_DMA_READ_REMOTE, }; union pcie_dma_ioctl_param { @@ -16,6 +19,7 @@ union pcie_dma_ioctl_param { u32 r_widx; u32 size; u32 type; + u32 chn; } in; struct { u32 lwa; @@ -23,6 +27,9 @@ union pcie_dma_ioctl_param { } out; u32 lra; u32 count; + u32 total_buffer_size; + phys_addr_t local_addr; + u32 buffer_size; }; #define PCIE_BASE 'P' @@ -42,4 +49,13 @@ union pcie_dma_ioctl_param { _IO(PCIE_BASE, 6) #define PCIE_DMA_SET_LOOP_COUNT \ _IOW(PCIE_BASE, 7, union pcie_dma_ioctl_param) +#define PCIE_DMA_GET_TOTAL_BUFFER_SIZE \ + _IOW(PCIE_BASE, 8, union pcie_dma_ioctl_param) +#define PCIE_DMA_SET_BUFFER_SIZE \ + _IOW(PCIE_BASE, 9, union pcie_dma_ioctl_param) +#define PCIE_DMA_READ_FROM_REMOTE \ + _IOW(PCIE_BASE, 0xa, union pcie_dma_ioctl_param) +#define PCIE_DMA_USER_SET_BUF_ADDR \ + _IOW(PCIE_BASE, 0xb, union pcie_dma_ioctl_param) +#endif