~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,6 +1,6 @@
1	1	// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
2	2	/*
3		- * Copyright (c) 2022 Rockchip Electronics Co., Ltd
	3	+ * Copyright (c) 2021 Rockchip Electronics Co., Ltd
4	4	*
5	5	* author:
6	6	* Ding Wei, leo.ding@rock-chips.com
..	..	@@ -41,6 +41,9 @@
41	41
42	42	#define RKVENC_SESSION_MAX_BUFFERS 40
43	43	#define RKVENC_MAX_CORE_NUM 4
	44	+#define RKVENC_MAX_DCHS_ID 4
	45	+#define RKVENC_MAX_SLICE_FIFO_LEN 256
	46	+#define RKVENC_SCLR_DONE_STA BIT(2)
44	47
45	48	#define to_rkvenc_info(info) \
46	49	container_of(info, struct rkvenc_hw_info, hw)
..	..	@@ -116,6 +119,82 @@
116	119	u32 err_mask;
117	120	};
118	121
	122	+#define INT_STA_ENC_DONE_STA BIT(0)
	123	+#define INT_STA_SCLR_DONE_STA BIT(2)
	124	+#define INT_STA_SLC_DONE_STA BIT(3)
	125	+#define INT_STA_BSF_OFLW_STA BIT(4)
	126	+#define INT_STA_BRSP_OTSD_STA BIT(5)
	127	+#define INT_STA_WBUS_ERR_STA BIT(6)
	128	+#define INT_STA_RBUS_ERR_STA BIT(7)
	129	+#define INT_STA_WDG_STA BIT(8)
	130	+
	131	+#define DCHS_REG_OFFSET (0x304)
	132	+#define DCHS_CLASS_OFFSET (33)
	133	+#define DCHS_TXE (0x10)
	134	+#define DCHS_RXE (0x20)
	135	+
	136	+/* dual core hand-shake info */
	137	+union rkvenc2_dual_core_handshake_id {
	138	+ u64 val;
	139	+ struct {
	140	+ u32 txid : 2;
	141	+ u32 rxid : 2;
	142	+ u32 txe : 1;
	143	+ u32 rxe : 1;
	144	+ u32 working : 1;
	145	+ u32 reserve0 : 1;
	146	+ u32 txid_orig : 2;
	147	+ u32 rxid_orig : 2;
	148	+ u32 txid_map : 2;
	149	+ u32 rxid_map : 2;
	150	+ u32 offset : 11;
	151	+ u32 reserve1 : 1;
	152	+ u32 txe_orig : 1;
	153	+ u32 rxe_orig : 1;
	154	+ u32 txe_map : 1;
	155	+ u32 rxe_map : 1;
	156	+ u32 session_id;
	157	+ };
	158	+};
	159	+
	160	+#define RKVENC2_REG_INT_EN (8)
	161	+#define RKVENC2_BIT_SLICE_DONE_EN BIT(3)
	162	+
	163	+#define RKVENC2_REG_INT_MASK (9)
	164	+#define RKVENC2_BIT_SLICE_DONE_MASK BIT(3)
	165	+
	166	+#define RKVENC2_REG_EXT_LINE_BUF_BASE (22)
	167	+
	168	+#define RKVENC2_REG_ENC_PIC (32)
	169	+#define RKVENC2_BIT_ENC_STND BIT(0)
	170	+#define RKVENC2_BIT_VAL_H264 0
	171	+#define RKVENC2_BIT_VAL_H265 1
	172	+#define RKVENC2_BIT_SLEN_FIFO BIT(30)
	173	+
	174	+#define RKVENC2_REG_SLI_SPLIT (56)
	175	+#define RKVENC2_BIT_SLI_SPLIT BIT(0)
	176	+#define RKVENC2_BIT_SLI_FLUSH BIT(15)
	177	+
	178	+#define RKVENC2_REG_SLICE_NUM_BASE (0x4034)
	179	+#define RKVENC2_REG_SLICE_LEN_BASE (0x4038)
	180	+
	181	+union rkvenc2_slice_len_info {
	182	+ u32 val;
	183	+
	184	+ struct {
	185	+ u32 slice_len : 31;
	186	+ u32 last : 1;
	187	+ };
	188	+};
	189	+
	190	+struct rkvenc_poll_slice_cfg {
	191	+ s32 poll_type;
	192	+ s32 poll_ret;
	193	+ s32 count_max;
	194	+ s32 count_ret;
	195	+ union rkvenc2_slice_len_info slice_info[];
	196	+};
	197	+
119	198	struct rkvenc_task {
120	199	struct mpp_task mpp_task;
121	200	int fmt;
..	..	@@ -138,7 +217,20 @@
138	217	u32 r_req_cnt;
139	218	struct mpp_request r_reqs[MPP_MAX_MSG_NUM];
140	219	struct mpp_dma_buffer *table;
141		- u32 task_no;
	220	+
	221	+ union rkvenc2_dual_core_handshake_id dchs_id;
	222	+
	223	+ /* split output / slice mode info */
	224	+ u32 task_split;
	225	+ u32 task_split_done;
	226	+ u32 last_slice_found;
	227	+ u32 slice_wr_cnt;
	228	+ u32 slice_rd_cnt;
	229	+ DECLARE_KFIFO(slice_info, union rkvenc2_slice_len_info, RKVENC_MAX_SLICE_FIFO_LEN);
	230	+
	231	+ /* jpege bitstream */
	232	+ struct mpp_dma_buffer *bs_buf;
	233	+ u32 offset_bs;
142	234	};
143	235
144	236	#define RKVENC_MAX_RCB_NUM (4)
..	..	@@ -180,6 +272,9 @@
180	272	struct reset_control *rst_a;
181	273	struct reset_control *rst_h;
182	274	struct reset_control *rst_core;
	275	+ /* for ccu */
	276	+ struct rkvenc_ccu *ccu;
	277	+ struct list_head core_link;
183	278
184	279	/* internal rcb-memory */
185	280	u32 sram_size;
..	..	@@ -187,10 +282,95 @@
187	282	dma_addr_t sram_iova;
188	283	u32 sram_enabled;
189	284	struct page *rcb_page;
	285	+
	286	+#ifdef CONFIG_PM_DEVFREQ
	287	+ struct rockchip_opp_info opp_info;
	288	+ struct monitor_dev_info *mdev_info;
	289	+#endif
190	290	};
191	291
	292	+struct rkvenc_ccu {
	293	+ u32 core_num;
	294	+ /* lock for core attach */
	295	+ struct mutex lock;
	296	+ struct list_head core_list;
	297	+ struct mpp_dev *main_core;
	298	+
	299	+ spinlock_t lock_dchs;
	300	+ union rkvenc2_dual_core_handshake_id dchs[RKVENC_MAX_CORE_NUM];
	301	+};
192	302
193	303	static struct rkvenc_hw_info rkvenc_v2_hw_info = {
	304	+ .hw = {
	305	+ .reg_num = 254,
	306	+ .reg_id = 0,
	307	+ .reg_en = 4,
	308	+ .reg_start = 160,
	309	+ .reg_end = 253,
	310	+ },
	311	+ .reg_class = RKVENC_CLASS_BUTT,
	312	+ .reg_msg[RKVENC_CLASS_BASE] = {
	313	+ .base_s = 0x0000,
	314	+ .base_e = 0x0058,
	315	+ },
	316	+ .reg_msg[RKVENC_CLASS_PIC] = {
	317	+ .base_s = 0x0280,
	318	+ .base_e = 0x03f4,
	319	+ },
	320	+ .reg_msg[RKVENC_CLASS_RC] = {
	321	+ .base_s = 0x1000,
	322	+ .base_e = 0x10e0,
	323	+ },
	324	+ .reg_msg[RKVENC_CLASS_PAR] = {
	325	+ .base_s = 0x1700,
	326	+ .base_e = 0x1cd4,
	327	+ },
	328	+ .reg_msg[RKVENC_CLASS_SQI] = {
	329	+ .base_s = 0x2000,
	330	+ .base_e = 0x21e4,
	331	+ },
	332	+ .reg_msg[RKVENC_CLASS_SCL] = {
	333	+ .base_s = 0x2200,
	334	+ .base_e = 0x2c98,
	335	+ },
	336	+ .reg_msg[RKVENC_CLASS_OSD] = {
	337	+ .base_s = 0x3000,
	338	+ .base_e = 0x347c,
	339	+ },
	340	+ .reg_msg[RKVENC_CLASS_ST] = {
	341	+ .base_s = 0x4000,
	342	+ .base_e = 0x42cc,
	343	+ },
	344	+ .reg_msg[RKVENC_CLASS_DEBUG] = {
	345	+ .base_s = 0x5000,
	346	+ .base_e = 0x5354,
	347	+ },
	348	+ .fd_class = RKVENC_CLASS_FD_BUTT,
	349	+ .fd_reg[RKVENC_CLASS_FD_BASE] = {
	350	+ .class = RKVENC_CLASS_PIC,
	351	+ .base_fmt = RKVENC_FMT_BASE,
	352	+ },
	353	+ .fd_reg[RKVENC_CLASS_FD_OSD] = {
	354	+ .class = RKVENC_CLASS_OSD,
	355	+ .base_fmt = RKVENC_FMT_OSD_BASE,
	356	+ },
	357	+ .fmt_reg = {
	358	+ .class = RKVENC_CLASS_PIC,
	359	+ .base = 0x0300,
	360	+ .bitpos = 0,
	361	+ .bitlen = 1,
	362	+ },
	363	+ .enc_start_base = 0x0010,
	364	+ .enc_clr_base = 0x0014,
	365	+ .int_en_base = 0x0020,
	366	+ .int_mask_base = 0x0024,
	367	+ .int_clr_base = 0x0028,
	368	+ .int_sta_base = 0x002c,
	369	+ .enc_wdg_base = 0x0038,
	370	+ .err_mask = 0x03f0,
	371	+};
	372	+
	373	+static struct rkvenc_hw_info rkvenc_540c_hw_info = {
194	374	.hw = {
195	375	.reg_num = 254,
196	376	.reg_id = 0,
..	..	@@ -259,28 +439,50 @@
259	439	.enc_wdg_base = 0x0038,
260	440	.err_mask = 0x27d0,
261	441	};
262		-
263	442	/*
264	443	* file handle translate information for v2
265	444	*/
266	445	static const u16 trans_tbl_h264e_v2[] = {
	446	+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
	447	+ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
	448	+ 20, 21, 22, 23,
	449	+};
	450	+
	451	+static const u16 trans_tbl_h264e_v2_osd[] = {
	452	+ 20, 21, 22, 23, 24, 25, 26, 27,
	453	+};
	454	+
	455	+static const u16 trans_tbl_h265e_v2[] = {
	456	+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
	457	+ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
	458	+ 20, 21, 22, 23,
	459	+};
	460	+
	461	+static const u16 trans_tbl_h265e_v2_osd[] = {
	462	+ 20, 21, 22, 23, 24, 25, 26, 27,
	463	+};
	464	+
	465	+/*
	466	+ * file handle translate information for 540c
	467	+ */
	468	+static const u16 trans_tbl_h264e_540c[] = {
267	469	4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
268	470	14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
269	471	// /* renc and ref wrap */
270	472	// 24, 25, 26, 27,
271	473	};
272	474
273		-static const u16 trans_tbl_h264e_v2_osd[] = {
	475	+static const u16 trans_tbl_h264e_540c_osd[] = {
274	476	3, 4, 12, 13, 21, 22, 30, 31,
275	477	39, 40, 48, 49, 57, 58, 66, 67,
276	478	};
277	479
278		-static const u16 trans_tbl_h265e_v2[] = {
	480	+static const u16 trans_tbl_h265e_540c[] = {
279	481	4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
280	482	14, 15, 16, 17, 18, 19, 20, 21, 22, 23
281	483	};
282	484
283		-static const u16 trans_tbl_h265e_v2_osd[] = {
	485	+static const u16 trans_tbl_h265e_540c_osd[] = {
284	486	3, 4, 12, 13, 21, 22, 30, 31,
285	487	39, 40, 48, 49, 57, 58, 66, 67,
286	488	};
..	..	@@ -311,6 +513,25 @@
311	513	[RKVENC_FMT_H265E_OSD] = {
312	514	.count = ARRAY_SIZE(trans_tbl_h265e_v2_osd),
313	515	.table = trans_tbl_h265e_v2_osd,
	516	+ },
	517	+};
	518	+
	519	+static struct mpp_trans_info trans_rkvenc_540c[] = {
	520	+ [RKVENC_FMT_H264E] = {
	521	+ .count = ARRAY_SIZE(trans_tbl_h264e_540c),
	522	+ .table = trans_tbl_h264e_540c,
	523	+ },
	524	+ [RKVENC_FMT_H264E_OSD] = {
	525	+ .count = ARRAY_SIZE(trans_tbl_h264e_540c_osd),
	526	+ .table = trans_tbl_h264e_540c_osd,
	527	+ },
	528	+ [RKVENC_FMT_H265E] = {
	529	+ .count = ARRAY_SIZE(trans_tbl_h265e_540c),
	530	+ .table = trans_tbl_h265e_540c,
	531	+ },
	532	+ [RKVENC_FMT_H265E_OSD] = {
	533	+ .count = ARRAY_SIZE(trans_tbl_h265e_540c_osd),
	534	+ .table = trans_tbl_h265e_540c_osd,
314	535	},
315	536	[RKVENC_FMT_JPEGE] = {
316	537	.count = ARRAY_SIZE(trans_tbl_jpege),
..	..	@@ -345,6 +566,7 @@
345	566
346	567	for (i = 0; i < reg_class; i++) {
347	568	kfree(task->reg[i].data);
	569	+ task->reg[i].data = NULL;
348	570	task->reg[i].size = 0;
349	571	}
350	572
..	..	@@ -570,7 +792,6 @@
570	792	return 0;
571	793	}
572	794
573		-
574	795	static int rkvenc2_set_rcbbuf(struct mpp_dev mpp, struct mpp_session session,
575	796	struct rkvenc_task *task)
576	797	{
..	..	@@ -616,6 +837,56 @@
616	837	return 0;
617	838	}
618	839
	840	+static void rkvenc2_setup_task_id(u32 session_id, struct rkvenc_task *task)
	841	+{
	842	+ u32 val = task->reg[RKVENC_CLASS_PIC].data[DCHS_CLASS_OFFSET];
	843	+
	844	+ /* always enable tx */
	845	+ val \|= DCHS_TXE;
	846	+
	847	+ task->reg[RKVENC_CLASS_PIC].data[DCHS_CLASS_OFFSET] = val;
	848	+ task->dchs_id.val = (((u64)session_id << 32) \| val);
	849	+
	850	+ task->dchs_id.txid_orig = task->dchs_id.txid;
	851	+ task->dchs_id.rxid_orig = task->dchs_id.rxid;
	852	+ task->dchs_id.txid_map = task->dchs_id.txid;
	853	+ task->dchs_id.rxid_map = task->dchs_id.rxid;
	854	+
	855	+ task->dchs_id.txe_orig = task->dchs_id.txe;
	856	+ task->dchs_id.rxe_orig = task->dchs_id.rxe;
	857	+ task->dchs_id.txe_map = task->dchs_id.txe;
	858	+ task->dchs_id.rxe_map = task->dchs_id.rxe;
	859	+}
	860	+
	861	+static void rkvenc2_check_split_task(struct rkvenc_task *task)
	862	+{
	863	+ u32 slen_fifo_en = 0;
	864	+ u32 sli_split_en = 0;
	865	+
	866	+ if (task->reg[RKVENC_CLASS_PIC].valid) {
	867	+ u32 *reg = task->reg[RKVENC_CLASS_PIC].data;
	868	+ u32 enc_stnd = reg[RKVENC2_REG_ENC_PIC] & RKVENC2_BIT_ENC_STND;
	869	+
	870	+ slen_fifo_en = (reg[RKVENC2_REG_ENC_PIC] & RKVENC2_BIT_SLEN_FIFO) ? 1 : 0;
	871	+ sli_split_en = (reg[RKVENC2_REG_SLI_SPLIT] & RKVENC2_BIT_SLI_SPLIT) ? 1 : 0;
	872	+
	873	+ /*
	874	+ * FIXUP: rkvenc2 hardware bug:
	875	+ * H.264 encoding has bug when external line buffer and slice flush both
	876	+ * are enabled.
	877	+ */
	878	+ if (sli_split_en && slen_fifo_en &&
	879	+ enc_stnd == RKVENC2_BIT_VAL_H264 &&
	880	+ reg[RKVENC2_REG_EXT_LINE_BUF_BASE])
	881	+ reg[RKVENC2_REG_SLI_SPLIT] &= ~RKVENC2_BIT_SLI_FLUSH;
	882	+ }
	883	+
	884	+ task->task_split = sli_split_en && slen_fifo_en;
	885	+
	886	+ if (task->task_split)
	887	+ INIT_KFIFO(task->slice_info);
	888	+}
	889	+
619	890	static void rkvenc_alloc_task(struct mpp_session session,
620	891	struct mpp_task_msgs *msgs)
621	892	{
..	..	@@ -650,6 +921,7 @@
650	921	u32 off;
651	922	const u16 *tbl;
652	923	struct rkvenc_hw_info *hw = task->hw_info;
	924	+ int fd_bs = -1;
653	925
654	926	for (i = 0; i < hw->fd_class; i++) {
655	927	u32 class = hw->fd_reg[i].class;
..	..	@@ -659,6 +931,15 @@
659	931
660	932	if (!reg)
661	933	continue;
	934	+
	935	+ if (fmt == RKVENC_FMT_JPEGE && class == RKVENC_CLASS_PIC && fd_bs == -1) {
	936	+ int bs_index;
	937	+
	938	+ bs_index = mpp->var->trans_info[fmt].table[2];
	939	+ fd_bs = reg[bs_index];
	940	+ task->offset_bs = mpp_query_reg_offset_info(&task->off_inf,
	941	+ bs_index + ss);
	942	+ }
662	943
663	944	ret = mpp_translate_reg_address(session, mpp_task, fmt, reg, NULL);
664	945	if (ret)
..	..	@@ -672,9 +953,19 @@
672	953	reg[tbl[j]] += off;
673	954	}
674	955	}
	956	+
	957	+ if (fd_bs >= 0) {
	958	+ struct mpp_dma_buffer *bs_buf =
	959	+ mpp_dma_find_buffer_fd(session->dma, fd_bs);
	960	+
	961	+ if (bs_buf && task->offset_bs > 0)
	962	+ mpp_dma_buf_sync(bs_buf, 0, task->offset_bs, DMA_TO_DEVICE, false);
	963	+ task->bs_buf = bs_buf;
	964	+ }
675	965	}
676		- rkvenc2_set_rcbbuf(mpp, session, task);
	966	+ rkvenc2_setup_task_id(session->index, task);
677	967	task->clk_mode = CLK_MODE_NORMAL;
	968	+ rkvenc2_check_split_task(task);
678	969
679	970	mpp_debug_leave();
680	971
..	..	@@ -692,6 +983,201 @@
692	983	return NULL;
693	984	}
694	985
	986	+static void rkvenc2_prepare(struct mpp_dev mpp, struct mpp_task *mpp_task)
	987	+{
	988	+ struct mpp_taskqueue *queue = mpp->queue;
	989	+ unsigned long core_idle;
	990	+ unsigned long flags;
	991	+ u32 core_id_max;
	992	+ s32 core_id;
	993	+ u32 i;
	994	+
	995	+ spin_lock_irqsave(&queue->running_lock, flags);
	996	+
	997	+ core_idle = queue->core_idle;
	998	+ core_id_max = queue->core_id_max;
	999	+
	1000	+ for (i = 0; i <= core_id_max; i++) {
	1001	+ struct mpp_dev *mpp = queue->cores[i];
	1002	+
	1003	+ if (mpp && mpp->disable)
	1004	+ clear_bit(i, &core_idle);
	1005	+ }
	1006	+
	1007	+ core_id = find_first_bit(&core_idle, core_id_max + 1);
	1008	+
	1009	+ if (core_id >= core_id_max + 1 \|\| !queue->cores[core_id]) {
	1010	+ mpp_task = NULL;
	1011	+ mpp_dbg_core("core %d all busy %lx\n", core_id, core_idle);
	1012	+ } else {
	1013	+ struct rkvenc_task *task = to_rkvenc_task(mpp_task);
	1014	+
	1015	+ clear_bit(core_id, &queue->core_idle);
	1016	+ mpp_task->mpp = queue->cores[core_id];
	1017	+ mpp_task->core_id = core_id;
	1018	+ rkvenc2_set_rcbbuf(mpp_task->mpp, mpp_task->session, task);
	1019	+ mpp_dbg_core("core %d set idle %lx -> %lx\n", core_id,
	1020	+ core_idle, queue->core_idle);
	1021	+ }
	1022	+
	1023	+ spin_unlock_irqrestore(&queue->running_lock, flags);
	1024	+
	1025	+ return mpp_task;
	1026	+}
	1027	+
	1028	+static void rkvenc2_patch_dchs(struct rkvenc_dev enc, struct rkvenc_task task)
	1029	+{
	1030	+ struct rkvenc_ccu *ccu;
	1031	+ union rkvenc2_dual_core_handshake_id *dchs;
	1032	+ union rkvenc2_dual_core_handshake_id *task_dchs = &task->dchs_id;
	1033	+ int core_num;
	1034	+ int core_id = enc->mpp.core_id;
	1035	+ unsigned long flags;
	1036	+ int i;
	1037	+
	1038	+ if (!enc->ccu)
	1039	+ return;
	1040	+
	1041	+ if (core_id >= RKVENC_MAX_CORE_NUM) {
	1042	+ dev_err(enc->mpp.dev, "invalid core id %d max %d\n",
	1043	+ core_id, RKVENC_MAX_CORE_NUM);
	1044	+ return;
	1045	+ }
	1046	+
	1047	+ ccu = enc->ccu;
	1048	+ dchs = ccu->dchs;
	1049	+ core_num = ccu->core_num;
	1050	+
	1051	+ spin_lock_irqsave(&ccu->lock_dchs, flags);
	1052	+
	1053	+ if (dchs[core_id].working) {
	1054	+ spin_unlock_irqrestore(&ccu->lock_dchs, flags);
	1055	+
	1056	+ mpp_err("can not config when core %d is still working\n", core_id);
	1057	+ return;
	1058	+ }
	1059	+
	1060	+ if (mpp_debug_unlikely(DEBUG_CORE))
	1061	+ pr_info("core tx:rx 0 %s %d:%d %d:%d -- 1 %s %d:%d %d:%d -- task %d %d:%d %d:%d\n",
	1062	+ dchs[0].working ? "work" : "idle",
	1063	+ dchs[0].txid, dchs[0].txe, dchs[0].rxid, dchs[0].rxe,
	1064	+ dchs[1].working ? "work" : "idle",
	1065	+ dchs[1].txid, dchs[1].txe, dchs[1].rxid, dchs[1].rxe,
	1066	+ core_id, task_dchs->txid, task_dchs->txe,
	1067	+ task_dchs->rxid, task_dchs->rxe);
	1068	+
	1069	+ /* always use new id as */
	1070	+ {
	1071	+ struct mpp_task *mpp_task = &task->mpp_task;
	1072	+ unsigned long id_valid = (unsigned long)-1;
	1073	+ int txid_map = -1;
	1074	+ int rxid_map = -1;
	1075	+
	1076	+ /* scan all used id */
	1077	+ for (i = 0; i < core_num; i++) {
	1078	+ if (!dchs[i].working)
	1079	+ continue;
	1080	+
	1081	+ clear_bit(dchs[i].txid_map, &id_valid);
	1082	+ clear_bit(dchs[i].rxid_map, &id_valid);
	1083	+ }
	1084	+
	1085	+ if (task_dchs->rxe) {
	1086	+ for (i = 0; i < core_num; i++) {
	1087	+ if (i == core_id)
	1088	+ continue;
	1089	+
	1090	+ if (!dchs[i].working)
	1091	+ continue;
	1092	+
	1093	+ if (task_dchs->session_id != dchs[i].session_id)
	1094	+ continue;
	1095	+
	1096	+ if (task_dchs->rxid_orig != dchs[i].txid_orig)
	1097	+ continue;
	1098	+
	1099	+ rxid_map = dchs[i].txid_map;
	1100	+ break;
	1101	+ }
	1102	+ }
	1103	+
	1104	+ txid_map = find_first_bit(&id_valid, RKVENC_MAX_DCHS_ID);
	1105	+ if (txid_map == RKVENC_MAX_DCHS_ID) {
	1106	+ spin_unlock_irqrestore(&ccu->lock_dchs, flags);
	1107	+
	1108	+ mpp_err("task %d:%d on core %d failed to find a txid\n",
	1109	+ mpp_task->session->pid, mpp_task->task_id,
	1110	+ mpp_task->core_id);
	1111	+ return;
	1112	+ }
	1113	+
	1114	+ clear_bit(txid_map, &id_valid);
	1115	+ task_dchs->txid_map = txid_map;
	1116	+
	1117	+ if (rxid_map < 0) {
	1118	+ rxid_map = find_first_bit(&id_valid, RKVENC_MAX_DCHS_ID);
	1119	+ if (rxid_map == RKVENC_MAX_DCHS_ID) {
	1120	+ spin_unlock_irqrestore(&ccu->lock_dchs, flags);
	1121	+
	1122	+ mpp_err("task %d:%d on core %d failed to find a rxid\n",
	1123	+ mpp_task->session->pid, mpp_task->task_id,
	1124	+ mpp_task->core_id);
	1125	+ return;
	1126	+ }
	1127	+
	1128	+ task_dchs->rxe_map = 0;
	1129	+ }
	1130	+
	1131	+ task_dchs->rxid_map = rxid_map;
	1132	+ }
	1133	+
	1134	+ task_dchs->txid = task_dchs->txid_map;
	1135	+ task_dchs->rxid = task_dchs->rxid_map;
	1136	+ task_dchs->rxe = task_dchs->rxe_map;
	1137	+
	1138	+ dchs[core_id].val = task_dchs->val;
	1139	+ task->reg[RKVENC_CLASS_PIC].data[DCHS_CLASS_OFFSET] = task_dchs->val;
	1140	+
	1141	+ dchs[core_id].working = 1;
	1142	+
	1143	+ spin_unlock_irqrestore(&ccu->lock_dchs, flags);
	1144	+}
	1145	+
	1146	+static void rkvenc2_update_dchs(struct rkvenc_dev enc, struct rkvenc_task task)
	1147	+{
	1148	+ struct rkvenc_ccu *ccu = enc->ccu;
	1149	+ int core_id = enc->mpp.core_id;
	1150	+ unsigned long flags;
	1151	+
	1152	+ if (!ccu)
	1153	+ return;
	1154	+
	1155	+ if (core_id >= RKVENC_MAX_CORE_NUM) {
	1156	+ dev_err(enc->mpp.dev, "invalid core id %d max %d\n",
	1157	+ core_id, RKVENC_MAX_CORE_NUM);
	1158	+ return;
	1159	+ }
	1160	+
	1161	+ spin_lock_irqsave(&ccu->lock_dchs, flags);
	1162	+ ccu->dchs[core_id].val = 0;
	1163	+
	1164	+ if (mpp_debug_unlikely(DEBUG_CORE)) {
	1165	+ union rkvenc2_dual_core_handshake_id *dchs = ccu->dchs;
	1166	+ union rkvenc2_dual_core_handshake_id *task_dchs = &task->dchs_id;
	1167	+
	1168	+ pr_info("core %d task done\n", core_id);
	1169	+ pr_info("core tx:rx 0 %s %d:%d %d:%d -- 1 %s %d:%d %d:%d -- task %d %d:%d %d:%d\n",
	1170	+ dchs[0].working ? "work" : "idle",
	1171	+ dchs[0].txid, dchs[0].txe, dchs[0].rxid, dchs[0].rxe,
	1172	+ dchs[1].working ? "work" : "idle",
	1173	+ dchs[1].txid, dchs[1].txe, dchs[1].rxid, dchs[1].rxe,
	1174	+ core_id, task_dchs->txid, task_dchs->txe,
	1175	+ task_dchs->rxid, task_dchs->rxe);
	1176	+ }
	1177	+
	1178	+ spin_unlock_irqrestore(&ccu->lock_dchs, flags);
	1179	+}
	1180	+
695	1181	static int rkvenc_run(struct mpp_dev mpp, struct mpp_task mpp_task)
696	1182	{
697	1183	u32 i, j;
..	..	@@ -699,6 +1185,7 @@
699	1185	struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
700	1186	struct rkvenc_task *task = to_rkvenc_task(mpp_task);
701	1187	struct rkvenc_hw_info *hw = enc->hw_info;
	1188	+ u32 timing_en = mpp->srv->timing_en;
702	1189
703	1190	mpp_debug_enter();
704	1191
..	..	@@ -709,6 +1196,8 @@
709	1196
710	1197	/* clear hardware counter */
711	1198	mpp_write_relaxed(mpp, 0x5300, 0x2);
	1199	+
	1200	+ rkvenc2_patch_dchs(enc, task);
712	1201
713	1202	for (i = 0; i < task->w_req_cnt; i++) {
714	1203	int ret;
..	..	@@ -735,35 +1224,113 @@
735	1224	}
736	1225	}
737	1226
	1227	+ if (mpp_debug_unlikely(DEBUG_CORE))
	1228	+ dev_info(mpp->dev, "core %d dchs %08x\n", mpp->core_id,
	1229	+ mpp_read_relaxed(&enc->mpp, DCHS_REG_OFFSET));
	1230	+
	1231	+ /* flush tlb before starting hardware */
	1232	+ mpp_iommu_flush_tlb(mpp->iommu_info);
	1233	+
738	1234	/* init current task */
739	1235	mpp->cur_task = mpp_task;
	1236	+
	1237	+ mpp_task_run_begin(mpp_task, timing_en, MPP_WORK_TIMEOUT_DELAY);
	1238	+
740	1239	/* Flush the register before the start the device */
741	1240	wmb();
	1241	+
742	1242	mpp_write(mpp, enc->hw_info->enc_start_base, start_val);
	1243	+
	1244	+ mpp_task_run_end(mpp_task, timing_en);
743	1245
744	1246	mpp_debug_leave();
745	1247
746	1248	return 0;
747	1249	}
748	1250
	1251	+static void rkvenc2_read_slice_len(struct mpp_dev mpp, struct rkvenc_task task)
	1252	+{
	1253	+ u32 last = mpp_read_relaxed(mpp, 0x002c) & INT_STA_ENC_DONE_STA;
	1254	+ u32 sli_num = mpp_read_relaxed(mpp, RKVENC2_REG_SLICE_NUM_BASE);
	1255	+ union rkvenc2_slice_len_info slice_info;
	1256	+ u32 task_id = task->mpp_task.task_id;
	1257	+ u32 i;
	1258	+
	1259	+ mpp_dbg_slice("task %d wr %3d len start %s\n", task_id,
	1260	+ sli_num, last ? "last" : "");
	1261	+
	1262	+ for (i = 0; i < sli_num; i++) {
	1263	+ slice_info.val = mpp_read_relaxed(mpp, RKVENC2_REG_SLICE_LEN_BASE);
	1264	+
	1265	+ if (last && i == sli_num - 1) {
	1266	+ task->last_slice_found = 1;
	1267	+ slice_info.last = 1;
	1268	+ }
	1269	+
	1270	+ mpp_dbg_slice("task %d wr %3d len %d %s\n", task_id,
	1271	+ task->slice_wr_cnt, slice_info.slice_len,
	1272	+ slice_info.last ? "last" : "");
	1273	+
	1274	+ kfifo_in(&task->slice_info, &slice_info, 1);
	1275	+ task->slice_wr_cnt++;
	1276	+ }
	1277	+
	1278	+ /* Fixup for async between last flag and slice number register */
	1279	+ if (last && !task->last_slice_found) {
	1280	+ mpp_dbg_slice("task %d mark last slice\n", task_id);
	1281	+ slice_info.last = 1;
	1282	+ slice_info.slice_len = 0;
	1283	+ kfifo_in(&task->slice_info, &slice_info, 1);
	1284	+ }
	1285	+}
	1286	+
749	1287	static int rkvenc_irq(struct mpp_dev *mpp)
750	1288	{
751	1289	struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
752	1290	struct rkvenc_hw_info *hw = enc->hw_info;
	1291	+ struct mpp_task *mpp_task = NULL;
	1292	+ struct rkvenc_task *task = NULL;
	1293	+ int ret = IRQ_NONE;
753	1294
754	1295	mpp_debug_enter();
755	1296
756	1297	mpp->irq_status = mpp_read(mpp, hw->int_sta_base);
757	1298	if (!mpp->irq_status)
758		- return IRQ_NONE;
759		- mpp_write(mpp, hw->int_mask_base, 0x100);
760		- mpp_write(mpp, hw->int_clr_base, 0xffffffff);
761		- udelay(5);
762		- mpp_write(mpp, hw->int_sta_base, 0);
	1299	+ return ret;
	1300	+
	1301	+ if (mpp->cur_task) {
	1302	+ mpp_task = mpp->cur_task;
	1303	+ task = to_rkvenc_task(mpp_task);
	1304	+ }
	1305	+
	1306	+ if (mpp->irq_status & INT_STA_ENC_DONE_STA) {
	1307	+ if (task) {
	1308	+ if (task->task_split)
	1309	+ rkvenc2_read_slice_len(mpp, task);
	1310	+
	1311	+ wake_up(&mpp_task->wait);
	1312	+ }
	1313	+
	1314	+ mpp_write(mpp, hw->int_mask_base, 0x100);
	1315	+ mpp_write(mpp, hw->int_clr_base, 0xffffffff);
	1316	+ udelay(5);
	1317	+ mpp_write(mpp, hw->int_sta_base, 0);
	1318	+
	1319	+ ret = IRQ_WAKE_THREAD;
	1320	+ } else if (mpp->irq_status & INT_STA_SLC_DONE_STA) {
	1321	+ if (task && task->task_split) {
	1322	+ mpp_time_part_diff(mpp_task);
	1323	+
	1324	+ rkvenc2_read_slice_len(mpp, task);
	1325	+ wake_up(&mpp_task->wait);
	1326	+ }
	1327	+
	1328	+ mpp_write(mpp, hw->int_clr_base, INT_STA_SLC_DONE_STA);
	1329	+ }
763	1330
764	1331	mpp_debug_leave();
765	1332
766		- return IRQ_WAKE_THREAD;
	1333	+ return ret;
767	1334	}
768	1335
769	1336	static int rkvenc_isr(struct mpp_dev *mpp)
..	..	@@ -771,6 +1338,8 @@
771	1338	struct rkvenc_task *task;
772	1339	struct mpp_task *mpp_task;
773	1340	struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
	1341	+ struct mpp_taskqueue *queue = mpp->queue;
	1342	+ unsigned long core_idle;
774	1343
775	1344	mpp_debug_enter();
776	1345
..	..	@@ -783,17 +1352,33 @@
783	1352	mpp_task = mpp->cur_task;
784	1353	mpp_time_diff(mpp_task);
785	1354	mpp->cur_task = NULL;
	1355	+
	1356	+ if (mpp_task->mpp && mpp_task->mpp != mpp)
	1357	+ dev_err(mpp->dev, "mismatch core dev %p:%p\n", mpp_task->mpp, mpp);
	1358	+
786	1359	task = to_rkvenc_task(mpp_task);
787	1360	task->irq_status = mpp->irq_status;
788		- mpp_debug(DEBUG_IRQ_STATUS, "irq_status: %08x\n", task->irq_status);
	1361	+
	1362	+ rkvenc2_update_dchs(enc, task);
	1363	+
	1364	+ mpp_debug(DEBUG_IRQ_STATUS, "%s irq_status: %08x\n",
	1365	+ dev_name(mpp->dev), task->irq_status);
789	1366
790	1367	if (task->irq_status & enc->hw_info->err_mask) {
791	1368	atomic_inc(&mpp->reset_request);
	1369	+
792	1370	/* dump register */
793	1371	if (mpp_debug_unlikely(DEBUG_DUMP_ERR_REG))
794		- mpp_task_dump_hw_reg(mpp, mpp_task);
	1372	+ mpp_task_dump_hw_reg(mpp);
795	1373	}
	1374	+
796	1375	mpp_task_finish(mpp_task->session, mpp_task);
	1376	+
	1377	+ core_idle = queue->core_idle;
	1378	+ set_bit(mpp->core_id, &queue->core_idle);
	1379	+
	1380	+ mpp_dbg_core("core %d isr idle %lx -> %lx\n", mpp->core_id, core_idle,
	1381	+ queue->core_idle);
797	1382
798	1383	mpp_debug_leave();
799	1384
..	..	@@ -824,6 +1409,14 @@
824	1409	reg[j] = mpp_read_relaxed(mpp, msg.offset + j * sizeof(u32));
825	1410
826	1411	}
	1412	+
	1413	+ if (task->bs_buf) {
	1414	+ u32 bs_size = mpp_read(mpp, 0x4064);
	1415	+
	1416	+ mpp_dma_buf_sync(task->bs_buf, 0, bs_size / 8 + task->offset_bs,
	1417	+ DMA_FROM_DEVICE, true);
	1418	+ }
	1419	+
827	1420	/* revert hack for irq status */
828	1421	reg = rkvenc_get_class_reg(task, task->hw_info->int_sta_base);
829	1422	if (reg)
..	..	@@ -975,7 +1568,7 @@
975	1568	}
976	1569	seq_puts(seq, "\n");
977	1570	/* item data*/
978		- seq_printf(seq, "\|%8p\|", session);
	1571	+ seq_printf(seq, "\|%8d\|", session->index);
979	1572	seq_printf(seq, "%8s\|", mpp_device_name[session->device_type]);
980	1573	for (i = ENC_INFO_BASE; i < ENC_INFO_BUTT; i++) {
981	1574	u32 flag = priv->codec_info[i].flag;
..	..	@@ -1008,7 +1601,7 @@
1008	1601	mutex_lock(&mpp->srv->session_lock);
1009	1602	list_for_each_entry_safe(session, n,
1010	1603	&mpp->srv->session_list,
1011		- session_link) {
	1604	+ service_link) {
1012	1605	if (session->device_type != MPP_DEVICE_RKVENC)
1013	1606	continue;
1014	1607	if (!session->priv)
..	..	@@ -1024,13 +1617,25 @@
1024	1617	static int rkvenc_procfs_init(struct mpp_dev *mpp)
1025	1618	{
1026	1619	struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
	1620	+ char name[32];
1027	1621
1028		- enc->procfs = proc_mkdir(mpp->dev->of_node->name, mpp->srv->procfs);
	1622	+ if (!mpp->dev \|\| !mpp->dev->of_node \|\| !mpp->dev->of_node->name \|\|
	1623	+ !mpp->srv \|\| !mpp->srv->procfs)
	1624	+ return -EINVAL;
	1625	+
	1626	+ snprintf(name, sizeof(name) - 1, "%s%d",
	1627	+ mpp->dev->of_node->name, mpp->core_id);
	1628	+
	1629	+ enc->procfs = proc_mkdir(name, mpp->srv->procfs);
1029	1630	if (IS_ERR_OR_NULL(enc->procfs)) {
1030	1631	mpp_err("failed on open procfs\n");
1031	1632	enc->procfs = NULL;
1032	1633	return -EIO;
1033	1634	}
	1635	+
	1636	+ /* for common mpp_dev options */
	1637	+ mpp_procfs_create_common(enc->procfs, mpp);
	1638	+
1034	1639	/* for debug */
1035	1640	mpp_procfs_create_u32("aclk", 0644,
1036	1641	enc->procfs, &enc->aclk_info.debug_rate_hz);
..	..	@@ -1045,6 +1650,16 @@
1045	1650	return 0;
1046	1651	}
1047	1652
	1653	+static int rkvenc_procfs_ccu_init(struct mpp_dev *mpp)
	1654	+{
	1655	+ struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
	1656	+
	1657	+ if (!enc->procfs)
	1658	+ goto done;
	1659	+
	1660	+done:
	1661	+ return 0;
	1662	+}
1048	1663	#else
1049	1664	static inline int rkvenc_procfs_remove(struct mpp_dev *mpp)
1050	1665	{
..	..	@@ -1056,6 +1671,105 @@
1056	1671	return 0;
1057	1672	}
1058	1673
	1674	+static inline int rkvenc_procfs_ccu_init(struct mpp_dev *mpp)
	1675	+{
	1676	+ return 0;
	1677	+}
	1678	+#endif
	1679	+
	1680	+#ifdef CONFIG_PM_DEVFREQ
	1681	+static int rk3588_venc_set_read_margin(struct device *dev,
	1682	+ struct rockchip_opp_info *opp_info,
	1683	+ u32 rm)
	1684	+{
	1685	+ if (!opp_info->grf \|\| !opp_info->volt_rm_tbl)
	1686	+ return 0;
	1687	+
	1688	+ if (rm == opp_info->current_rm \|\| rm == UINT_MAX)
	1689	+ return 0;
	1690	+
	1691	+ dev_dbg(dev, "set rm to %d\n", rm);
	1692	+
	1693	+ regmap_write(opp_info->grf, 0x214, 0x001c0000 \| (rm << 2));
	1694	+ regmap_write(opp_info->grf, 0x218, 0x001c0000 \| (rm << 2));
	1695	+ regmap_write(opp_info->grf, 0x220, 0x003c0000 \| (rm << 2));
	1696	+ regmap_write(opp_info->grf, 0x224, 0x003c0000 \| (rm << 2));
	1697	+
	1698	+ opp_info->current_rm = rm;
	1699	+
	1700	+ return 0;
	1701	+}
	1702	+
	1703	+static const struct rockchip_opp_data rk3588_venc_opp_data = {
	1704	+ .set_read_margin = rk3588_venc_set_read_margin,
	1705	+};
	1706	+
	1707	+static const struct of_device_id rockchip_rkvenc_of_match[] = {
	1708	+ {
	1709	+ .compatible = "rockchip,rk3588",
	1710	+ .data = (void *)&rk3588_venc_opp_data,
	1711	+ },
	1712	+ {},
	1713	+};
	1714	+
	1715	+static struct monitor_dev_profile venc_mdevp = {
	1716	+ .type = MONITOR_TYPE_DEV,
	1717	+ .update_volt = rockchip_monitor_check_rate_volt,
	1718	+};
	1719	+
	1720	+static int rkvenc_devfreq_init(struct mpp_dev *mpp)
	1721	+{
	1722	+ struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
	1723	+ struct clk *clk_core = enc->core_clk_info.clk;
	1724	+ struct device *dev = mpp->dev;
	1725	+ struct opp_table *reg_table = NULL;
	1726	+ struct opp_table *clk_table = NULL;
	1727	+ const char *const reg_names[] = { "venc", "mem" };
	1728	+ int ret = 0;
	1729	+
	1730	+ if (!clk_core)
	1731	+ return 0;
	1732	+
	1733	+ if (of_find_property(dev->of_node, "venc-supply", NULL) &&
	1734	+ of_find_property(dev->of_node, "mem-supply", NULL)) {
	1735	+ reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
	1736	+ if (IS_ERR(reg_table))
	1737	+ return PTR_ERR(reg_table);
	1738	+ } else {
	1739	+ reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
	1740	+ if (IS_ERR(reg_table))
	1741	+ return PTR_ERR(reg_table);
	1742	+ }
	1743	+
	1744	+ clk_table = dev_pm_opp_set_clkname(dev, "clk_core");
	1745	+ if (IS_ERR(clk_table))
	1746	+ return PTR_ERR(clk_table);
	1747	+
	1748	+ rockchip_get_opp_data(rockchip_rkvenc_of_match, &enc->opp_info);
	1749	+ ret = rockchip_init_opp_table(dev, &enc->opp_info, "leakage", "venc");
	1750	+ if (ret) {
	1751	+ dev_err(dev, "failed to init_opp_table\n");
	1752	+ return ret;
	1753	+ }
	1754	+
	1755	+ enc->mdev_info = rockchip_system_monitor_register(dev, &venc_mdevp);
	1756	+ if (IS_ERR(enc->mdev_info)) {
	1757	+ dev_dbg(dev, "without system monitor\n");
	1758	+ enc->mdev_info = NULL;
	1759	+ }
	1760	+
	1761	+ return ret;
	1762	+}
	1763	+
	1764	+static int rkvenc_devfreq_remove(struct mpp_dev *mpp)
	1765	+{
	1766	+ struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
	1767	+
	1768	+ if (enc->mdev_info)
	1769	+ rockchip_system_monitor_unregister(enc->mdev_info);
	1770	+
	1771	+ return 0;
	1772	+}
1059	1773	#endif
1060	1774
1061	1775	static int rkvenc_init(struct mpp_dev *mpp)
..	..	@@ -1094,26 +1808,60 @@
1094	1808	if (!enc->rst_core)
1095	1809	mpp_err("No core reset resource define\n");
1096	1810
	1811	+#ifdef CONFIG_PM_DEVFREQ
	1812	+ ret = rkvenc_devfreq_init(mpp);
	1813	+ if (ret)
	1814	+ mpp_err("failed to add venc devfreq\n");
	1815	+#endif
	1816	+
1097	1817	return 0;
	1818	+}
	1819	+
	1820	+static int rkvenc_exit(struct mpp_dev *mpp)
	1821	+{
	1822	+#ifdef CONFIG_PM_DEVFREQ
	1823	+ rkvenc_devfreq_remove(mpp);
	1824	+#endif
	1825	+
	1826	+ return 0;
	1827	+}
	1828	+
	1829	+static int rkvenc_soft_reset(struct mpp_dev *mpp)
	1830	+{
	1831	+ struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
	1832	+ struct rkvenc_hw_info *hw = enc->hw_info;
	1833	+ u32 rst_status = 0;
	1834	+ int ret = 0;
	1835	+
	1836	+ /* safe reset */
	1837	+ mpp_write(mpp, hw->int_mask_base, 0x3FF);
	1838	+ mpp_write(mpp, hw->enc_clr_base, 0x1);
	1839	+ ret = readl_relaxed_poll_timeout(mpp->reg_base + hw->int_sta_base,
	1840	+ rst_status,
	1841	+ rst_status & RKVENC_SCLR_DONE_STA,
	1842	+ 0, 5);
	1843	+ mpp_write(mpp, hw->int_clr_base, 0xffffffff);
	1844	+ mpp_write(mpp, hw->int_sta_base, 0);
	1845	+
	1846	+ return ret;
	1847	+
1098	1848	}
1099	1849
1100	1850	static int rkvenc_reset(struct mpp_dev *mpp)
1101	1851	{
1102	1852	struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
1103		- struct rkvenc_hw_info *hw = enc->hw_info;
	1853	+ int ret = 0;
	1854	+ struct mpp_taskqueue *queue = mpp->queue;
1104	1855
1105	1856	mpp_debug_enter();
1106	1857
1107		- /* safe reset */
1108		- mpp_write(mpp, hw->int_mask_base, 0x3FF);
1109		- mpp_write(mpp, hw->enc_clr_base, 0x1);
1110		- udelay(5);
1111		- mpp_write(mpp, hw->int_clr_base, 0xffffffff);
1112		- mpp_write(mpp, hw->int_sta_base, 0);
	1858	+ /* safe reset first*/
	1859	+ ret = rkvenc_soft_reset(mpp);
1113	1860
1114	1861	/* cru reset */
1115		- if (enc->rst_a && enc->rst_h && enc->rst_core) {
1116		- rockchip_pmu_idle_request(mpp->dev, true);
	1862	+ if (ret && enc->rst_a && enc->rst_h && enc->rst_core) {
	1863	+ mpp_err("soft reset timeout, use cru reset\n");
	1864	+ mpp_pmu_idle_request(mpp, true);
1117	1865	mpp_safe_reset(enc->rst_a);
1118	1866	mpp_safe_reset(enc->rst_h);
1119	1867	mpp_safe_reset(enc->rst_core);
..	..	@@ -1121,8 +1869,14 @@
1121	1869	mpp_safe_unreset(enc->rst_a);
1122	1870	mpp_safe_unreset(enc->rst_h);
1123	1871	mpp_safe_unreset(enc->rst_core);
1124		- rockchip_pmu_idle_request(mpp->dev, false);
	1872	+ mpp_pmu_idle_request(mpp, false);
1125	1873	}
	1874	+
	1875	+ set_bit(mpp->core_id, &queue->core_idle);
	1876	+ if (enc->ccu)
	1877	+ enc->ccu->dchs[mpp->core_id].val = 0;
	1878	+
	1879	+ mpp_dbg_core("core %d reset idle %lx\n", mpp->core_id, queue->core_idle);
1126	1880
1127	1881	mpp_debug_leave();
1128	1882
..	..	@@ -1162,8 +1916,183 @@
1162	1916	return 0;
1163	1917	}
1164	1918
	1919	+#define RKVENC2_WORK_TIMEOUT_DELAY (200)
	1920	+#define RKVENC2_WAIT_TIMEOUT_DELAY (2000)
	1921	+
	1922	+static void rkvenc2_task_pop_pending(struct mpp_task *task)
	1923	+{
	1924	+ struct mpp_session *session = task->session;
	1925	+
	1926	+ mutex_lock(&session->pending_lock);
	1927	+ list_del_init(&task->pending_link);
	1928	+ mutex_unlock(&session->pending_lock);
	1929	+
	1930	+ kref_put(&task->ref, mpp_free_task);
	1931	+}
	1932	+
	1933	+static int rkvenc2_task_default_process(struct mpp_dev *mpp,
	1934	+ struct mpp_task *task)
	1935	+{
	1936	+ int ret = 0;
	1937	+
	1938	+ if (mpp->dev_ops && mpp->dev_ops->result)
	1939	+ ret = mpp->dev_ops->result(mpp, task, NULL);
	1940	+
	1941	+ mpp_debug_func(DEBUG_TASK_INFO, "kref_read %d, ret %d\n",
	1942	+ kref_read(&task->ref), ret);
	1943	+
	1944	+ rkvenc2_task_pop_pending(task);
	1945	+
	1946	+ return ret;
	1947	+}
	1948	+
	1949	+#define RKVENC2_TIMEOUT_DUMP_REG_START (0x5100)
	1950	+#define RKVENC2_TIMEOUT_DUMP_REG_END (0x5160)
	1951	+
	1952	+static void rkvenc2_task_timeout_process(struct mpp_session *session,
	1953	+ struct mpp_task *task)
	1954	+{
	1955	+ atomic_inc(&task->abort_request);
	1956	+ set_bit(TASK_STATE_ABORT, &task->state);
	1957	+
	1958	+ mpp_err("session %d:%d count %d task %d ref %d timeout\n",
	1959	+ session->pid, session->index, atomic_read(&session->task_count),
	1960	+ task->task_id, kref_read(&task->ref));
	1961	+
	1962	+ if (task->mpp) {
	1963	+ struct mpp_dev *mpp = task->mpp;
	1964	+ u32 start = RKVENC2_TIMEOUT_DUMP_REG_START;
	1965	+ u32 end = RKVENC2_TIMEOUT_DUMP_REG_END;
	1966	+ u32 offset;
	1967	+
	1968	+ dev_err(mpp->dev, "core %d dump timeout status:\n", mpp->core_id);
	1969	+
	1970	+ for (offset = start; offset < end; offset += sizeof(u32))
	1971	+ mpp_reg_show(mpp, offset);
	1972	+ }
	1973	+
	1974	+ rkvenc2_task_pop_pending(task);
	1975	+}
	1976	+
	1977	+static int rkvenc2_wait_result(struct mpp_session *session,
	1978	+ struct mpp_task_msgs *msgs)
	1979	+{
	1980	+ struct rkvenc_poll_slice_cfg cfg;
	1981	+ struct rkvenc_task *enc_task;
	1982	+ struct mpp_request *req;
	1983	+ struct mpp_task *task;
	1984	+ struct mpp_dev *mpp;
	1985	+ union rkvenc2_slice_len_info slice_info;
	1986	+ u32 task_id;
	1987	+ int ret = 0;
	1988	+
	1989	+ mutex_lock(&session->pending_lock);
	1990	+ task = list_first_entry_or_null(&session->pending_list,
	1991	+ struct mpp_task,
	1992	+ pending_link);
	1993	+ mutex_unlock(&session->pending_lock);
	1994	+ if (!task) {
	1995	+ mpp_err("session %p pending list is empty!\n", session);
	1996	+ return -EIO;
	1997	+ }
	1998	+
	1999	+ mpp = mpp_get_task_used_device(task, session);
	2000	+ enc_task = to_rkvenc_task(task);
	2001	+ task_id = task->task_id;
	2002	+
	2003	+ req = cmpxchg(&msgs->poll_req, msgs->poll_req, NULL);
	2004	+
	2005	+ if (!enc_task->task_split \|\| enc_task->task_split_done) {
	2006	+task_done_ret:
	2007	+ ret = wait_event_timeout(task->wait,
	2008	+ test_bit(TASK_STATE_DONE, &task->state),
	2009	+ msecs_to_jiffies(RKVENC2_WAIT_TIMEOUT_DELAY));
	2010	+
	2011	+ if (ret > 0)
	2012	+ return rkvenc2_task_default_process(mpp, task);
	2013	+
	2014	+ rkvenc2_task_timeout_process(session, task);
	2015	+ return ret;
	2016	+ }
	2017	+
	2018	+ /* not slice return just wait all slice length */
	2019	+ if (!req) {
	2020	+ do {
	2021	+ ret = wait_event_timeout(task->wait,
	2022	+ kfifo_out(&enc_task->slice_info, &slice_info, 1),
	2023	+ msecs_to_jiffies(RKVENC2_WORK_TIMEOUT_DELAY));
	2024	+ if (ret > 0) {
	2025	+ mpp_dbg_slice("task %d rd %3d len %d %s\n",
	2026	+ task_id, enc_task->slice_rd_cnt, slice_info.slice_len,
	2027	+ slice_info.last ? "last" : "");
	2028	+
	2029	+ enc_task->slice_rd_cnt++;
	2030	+
	2031	+ if (slice_info.last)
	2032	+ goto task_done_ret;
	2033	+
	2034	+ continue;
	2035	+ }
	2036	+
	2037	+ rkvenc2_task_timeout_process(session, task);
	2038	+ return ret;
	2039	+ } while (1);
	2040	+ }
	2041	+
	2042	+ if (copy_from_user(&cfg, req->data, sizeof(cfg))) {
	2043	+ mpp_err("copy_from_user failed\n");
	2044	+ return -EINVAL;
	2045	+ }
	2046	+
	2047	+ mpp_dbg_slice("task %d poll irq %d:%d\n", task->task_id,
	2048	+ cfg.count_max, cfg.count_ret);
	2049	+ cfg.count_ret = 0;
	2050	+
	2051	+ /* handle slice mode poll return */
	2052	+ do {
	2053	+ ret = wait_event_timeout(task->wait,
	2054	+ kfifo_out(&enc_task->slice_info, &slice_info, 1),
	2055	+ msecs_to_jiffies(RKVENC2_WORK_TIMEOUT_DELAY));
	2056	+ if (ret > 0) {
	2057	+ mpp_dbg_slice("core %d task %d rd %3d len %d %s\n", task_id,
	2058	+ mpp->core_id, enc_task->slice_rd_cnt, slice_info.slice_len,
	2059	+ slice_info.last ? "last" : "");
	2060	+ enc_task->slice_rd_cnt++;
	2061	+ if (cfg.count_ret < cfg.count_max) {
	2062	+ struct rkvenc_poll_slice_cfg __user *ucfg =
	2063	+ (struct rkvenc_poll_slice_cfg __user *)(req->data);
	2064	+ u32 __user dst = (u32 __user )(ucfg + 1);
	2065	+
	2066	+ /* Do NOT return here when put_user error. Just continue */
	2067	+ if (put_user(slice_info.val, dst + cfg.count_ret))
	2068	+ ret = -EFAULT;
	2069	+
	2070	+ cfg.count_ret++;
	2071	+ if (put_user(cfg.count_ret, &ucfg->count_ret))
	2072	+ ret = -EFAULT;
	2073	+ }
	2074	+
	2075	+ if (slice_info.last) {
	2076	+ enc_task->task_split_done = 1;
	2077	+ goto task_done_ret;
	2078	+ }
	2079	+
	2080	+ if (cfg.count_ret >= cfg.count_max)
	2081	+ return 0;
	2082	+
	2083	+ if (ret < 0)
	2084	+ return ret;
	2085	+ }
	2086	+ } while (ret > 0);
	2087	+
	2088	+ rkvenc2_task_timeout_process(session, task);
	2089	+
	2090	+ return ret;
	2091	+}
	2092	+
1165	2093	static struct mpp_hw_ops rkvenc_hw_ops = {
1166	2094	.init = rkvenc_init,
	2095	+ .exit = rkvenc_exit,
1167	2096	.clk_on = rkvenc_clk_on,
1168	2097	.clk_off = rkvenc_clk_off,
1169	2098	.set_freq = rkvenc_set_freq,
..	..	@@ -1171,7 +2100,24 @@
1171	2100	};
1172	2101
1173	2102	static struct mpp_dev_ops rkvenc_dev_ops_v2 = {
	2103	+ .wait_result = rkvenc2_wait_result,
1174	2104	.alloc_task = rkvenc_alloc_task,
	2105	+ .run = rkvenc_run,
	2106	+ .irq = rkvenc_irq,
	2107	+ .isr = rkvenc_isr,
	2108	+ .finish = rkvenc_finish,
	2109	+ .result = rkvenc_result,
	2110	+ .free_task = rkvenc_free_task,
	2111	+ .ioctl = rkvenc_control,
	2112	+ .init_session = rkvenc_init_session,
	2113	+ .free_session = rkvenc_free_session,
	2114	+ .dump_session = rkvenc_dump_session,
	2115	+};
	2116	+
	2117	+static struct mpp_dev_ops rkvenc_ccu_dev_ops = {
	2118	+ .wait_result = rkvenc2_wait_result,
	2119	+ .alloc_task = rkvenc_alloc_task,
	2120	+ .prepare = rkvenc2_prepare,
1175	2121	.run = rkvenc_run,
1176	2122	.irq = rkvenc_irq,
1177	2123	.isr = rkvenc_isr,
..	..	@@ -1193,13 +2139,125 @@
1193	2139	.dev_ops = &rkvenc_dev_ops_v2,
1194	2140	};
1195	2141
	2142	+static const struct mpp_dev_var rkvenc_540c_data = {
	2143	+ .device_type = MPP_DEVICE_RKVENC,
	2144	+ .hw_info = &rkvenc_540c_hw_info.hw,
	2145	+ .trans_info = trans_rkvenc_540c,
	2146	+ .hw_ops = &rkvenc_hw_ops,
	2147	+ .dev_ops = &rkvenc_dev_ops_v2,
	2148	+};
	2149	+
	2150	+static const struct mpp_dev_var rkvenc_ccu_data = {
	2151	+ .device_type = MPP_DEVICE_RKVENC,
	2152	+ .hw_info = &rkvenc_v2_hw_info.hw,
	2153	+ .trans_info = trans_rkvenc_v2,
	2154	+ .hw_ops = &rkvenc_hw_ops,
	2155	+ .dev_ops = &rkvenc_ccu_dev_ops,
	2156	+};
	2157	+
1196	2158	static const struct of_device_id mpp_rkvenc_dt_match[] = {
1197	2159	{
1198	2160	.compatible = "rockchip,rkv-encoder-v2",
1199	2161	.data = &rkvenc_v2_data,
1200	2162	},
	2163	+#ifdef CONFIG_CPU_RK3528
	2164	+ {
	2165	+ .compatible = "rockchip,rkv-encoder-rk3528",
	2166	+ .data = &rkvenc_540c_data,
	2167	+ },
	2168	+#endif
	2169	+#ifdef CONFIG_CPU_RK3562
	2170	+ {
	2171	+ .compatible = "rockchip,rkv-encoder-rk3562",
	2172	+ .data = &rkvenc_540c_data,
	2173	+ },
	2174	+#endif
	2175	+#ifdef CONFIG_CPU_RK3588
	2176	+ {
	2177	+ .compatible = "rockchip,rkv-encoder-v2-core",
	2178	+ .data = &rkvenc_ccu_data,
	2179	+ },
	2180	+ {
	2181	+ .compatible = "rockchip,rkv-encoder-v2-ccu",
	2182	+ },
	2183	+#endif
1201	2184	{},
1202	2185	};
	2186	+
	2187	+static int rkvenc_ccu_probe(struct platform_device *pdev)
	2188	+{
	2189	+ struct rkvenc_ccu *ccu;
	2190	+ struct device *dev = &pdev->dev;
	2191	+
	2192	+ ccu = devm_kzalloc(dev, sizeof(*ccu), GFP_KERNEL);
	2193	+ if (!ccu)
	2194	+ return -ENOMEM;
	2195	+
	2196	+ platform_set_drvdata(pdev, ccu);
	2197	+
	2198	+ mutex_init(&ccu->lock);
	2199	+ INIT_LIST_HEAD(&ccu->core_list);
	2200	+ spin_lock_init(&ccu->lock_dchs);
	2201	+
	2202	+ return 0;
	2203	+}
	2204	+
	2205	+static int rkvenc_attach_ccu(struct device dev, struct rkvenc_dev enc)
	2206	+{
	2207	+ struct device_node *np;
	2208	+ struct platform_device *pdev;
	2209	+ struct rkvenc_ccu *ccu;
	2210	+
	2211	+ mpp_debug_enter();
	2212	+
	2213	+ np = of_parse_phandle(dev->of_node, "rockchip,ccu", 0);
	2214	+ if (!np \|\| !of_device_is_available(np))
	2215	+ return -ENODEV;
	2216	+
	2217	+ pdev = of_find_device_by_node(np);
	2218	+ of_node_put(np);
	2219	+ if (!pdev)
	2220	+ return -ENODEV;
	2221	+
	2222	+ ccu = platform_get_drvdata(pdev);
	2223	+ if (!ccu)
	2224	+ return -ENOMEM;
	2225	+
	2226	+ INIT_LIST_HEAD(&enc->core_link);
	2227	+ mutex_lock(&ccu->lock);
	2228	+ ccu->core_num++;
	2229	+ list_add_tail(&enc->core_link, &ccu->core_list);
	2230	+ mutex_unlock(&ccu->lock);
	2231	+
	2232	+ /* attach the ccu-domain to current core */
	2233	+ if (!ccu->main_core) {
	2234	+ /**
	2235	+ * set the first device for the main-core,
	2236	+ * then the domain of the main-core named ccu-domain
	2237	+ */
	2238	+ ccu->main_core = &enc->mpp;
	2239	+ } else {
	2240	+ struct mpp_iommu_info ccu_info, cur_info;
	2241	+
	2242	+ /* set the ccu-domain for current device */
	2243	+ ccu_info = ccu->main_core->iommu_info;
	2244	+ cur_info = enc->mpp.iommu_info;
	2245	+
	2246	+ cur_info->domain = ccu_info->domain;
	2247	+ cur_info->rw_sem = ccu_info->rw_sem;
	2248	+ mpp_iommu_attach(cur_info);
	2249	+
	2250	+ /* increase main core message capacity */
	2251	+ ccu->main_core->msgs_cap++;
	2252	+ enc->mpp.msgs_cap = 0;
	2253	+ }
	2254	+ enc->ccu = ccu;
	2255	+
	2256	+ dev_info(dev, "attach ccu as core %d\n", enc->mpp.core_id);
	2257	+ mpp_debug_enter();
	2258	+
	2259	+ return 0;
	2260	+}
1203	2261
1204	2262	static int rkvenc2_alloc_rcbbuf(struct platform_device pdev, struct rkvenc_dev enc)
1205	2263	{
..	..	@@ -1215,10 +2273,9 @@
1215	2273
1216	2274	/* get rcb iova start and size */
1217	2275	ret = device_property_read_u32_array(dev, "rockchip,rcb-iova", vals, 2);
1218		- if (ret) {
1219		- dev_err(dev, "could not find property rcb-iova\n");
	2276	+ if (ret)
1220	2277	return ret;
1221		- }
	2278	+
1222	2279	iova = PAGE_ALIGN(vals[0]);
1223	2280	sram_used = PAGE_ALIGN(vals[1]);
1224	2281	if (!sram_used) {
..	..	@@ -1300,6 +2357,82 @@
1300	2357	return ret;
1301	2358	}
1302	2359
	2360	+static int rkvenc2_iommu_fault_handle(struct iommu_domain *iommu,
	2361	+ struct device *iommu_dev,
	2362	+ unsigned long iova, int status, void *arg)
	2363	+{
	2364	+ struct mpp_dev mpp = (struct mpp_dev )arg;
	2365	+ struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
	2366	+ struct mpp_task *mpp_task = mpp->cur_task;
	2367	+
	2368	+ dev_info(mpp->dev, "core %d page fault found dchs %08x\n",
	2369	+ mpp->core_id, mpp_read_relaxed(&enc->mpp, DCHS_REG_OFFSET));
	2370	+
	2371	+ if (mpp_task)
	2372	+ mpp_task_dump_mem_region(mpp, mpp_task);
	2373	+
	2374	+ return 0;
	2375	+}
	2376	+
	2377	+static int rkvenc_core_probe(struct platform_device *pdev)
	2378	+{
	2379	+ int ret = 0;
	2380	+ struct device *dev = &pdev->dev;
	2381	+ struct rkvenc_dev *enc = NULL;
	2382	+ struct mpp_dev *mpp = NULL;
	2383	+
	2384	+ enc = devm_kzalloc(dev, sizeof(*enc), GFP_KERNEL);
	2385	+ if (!enc)
	2386	+ return -ENOMEM;
	2387	+
	2388	+ mpp = &enc->mpp;
	2389	+ platform_set_drvdata(pdev, mpp);
	2390	+
	2391	+ if (pdev->dev.of_node) {
	2392	+ struct device_node *np = pdev->dev.of_node;
	2393	+ const struct of_device_id *match = NULL;
	2394	+
	2395	+ match = of_match_node(mpp_rkvenc_dt_match, np);
	2396	+ if (match)
	2397	+ mpp->var = (struct mpp_dev_var *)match->data;
	2398	+
	2399	+ mpp->core_id = of_alias_get_id(np, "rkvenc");
	2400	+ }
	2401	+
	2402	+ ret = mpp_dev_probe(mpp, pdev);
	2403	+ if (ret)
	2404	+ return ret;
	2405	+
	2406	+ /* attach core to ccu */
	2407	+ ret = rkvenc_attach_ccu(dev, enc);
	2408	+ if (ret) {
	2409	+ dev_err(dev, "attach ccu failed\n");
	2410	+ return ret;
	2411	+ }
	2412	+ rkvenc2_alloc_rcbbuf(pdev, enc);
	2413	+
	2414	+ ret = devm_request_threaded_irq(dev, mpp->irq,
	2415	+ mpp_dev_irq,
	2416	+ mpp_dev_isr_sched,
	2417	+ IRQF_SHARED,
	2418	+ dev_name(dev), mpp);
	2419	+ if (ret) {
	2420	+ dev_err(dev, "register interrupter runtime failed\n");
	2421	+ return -EINVAL;
	2422	+ }
	2423	+ mpp->session_max_buffers = RKVENC_SESSION_MAX_BUFFERS;
	2424	+ enc->hw_info = to_rkvenc_info(mpp->var->hw_info);
	2425	+ mpp->iommu_info->hdl = rkvenc2_iommu_fault_handle;
	2426	+ rkvenc_procfs_init(mpp);
	2427	+ rkvenc_procfs_ccu_init(mpp);
	2428	+
	2429	+ /* if current is main-core, register current device to mpp service */
	2430	+ if (mpp == enc->ccu->main_core)
	2431	+ mpp_dev_register_srv(mpp, mpp->srv);
	2432	+
	2433	+ return 0;
	2434	+}
	2435	+
1303	2436	static int rkvenc_probe_default(struct platform_device *pdev)
1304	2437	{
1305	2438	int ret = 0;
..	..	@@ -1313,7 +2446,7 @@
1313	2446	return -ENOMEM;
1314	2447
1315	2448	mpp = &enc->mpp;
1316		- platform_set_drvdata(pdev, enc);
	2449	+ platform_set_drvdata(pdev, mpp);
1317	2450
1318	2451	if (pdev->dev.of_node) {
1319	2452	match = of_match_node(mpp_rkvenc_dt_match, pdev->dev.of_node);
..	..	@@ -1353,10 +2486,16 @@
1353	2486	{
1354	2487	int ret = 0;
1355	2488	struct device *dev = &pdev->dev;
	2489	+ struct device_node *np = dev->of_node;
1356	2490
1357	2491	dev_info(dev, "probing start\n");
1358	2492
1359		- ret = rkvenc_probe_default(pdev);
	2493	+ if (strstr(np->name, "ccu"))
	2494	+ ret = rkvenc_ccu_probe(pdev);
	2495	+ else if (strstr(np->name, "core"))
	2496	+ ret = rkvenc_core_probe(pdev);
	2497	+ else
	2498	+ ret = rkvenc_probe_default(pdev);
1360	2499
1361	2500	dev_info(dev, "probing finish\n");
1362	2501
..	..	@@ -1369,8 +2508,9 @@
1369	2508
1370	2509	if (enc->rcb_page) {
1371	2510	size_t page_size = PAGE_ALIGN(enc->sram_used - enc->sram_size);
	2511	+ int order = min(get_order(page_size), MAX_ORDER);
1372	2512
1373		- __free_pages(enc->rcb_page, get_order(page_size));
	2513	+ __free_pages(enc->rcb_page, order);
1374	2514	}
1375	2515	if (enc->sram_iova) {
1376	2516	domain = enc->mpp.iommu_info->domain;
..	..	@@ -1383,13 +2523,33 @@
1383	2523	static int rkvenc_remove(struct platform_device *pdev)
1384	2524	{
1385	2525	struct device *dev = &pdev->dev;
	2526	+ struct device_node *np = dev->of_node;
1386	2527
1387		- struct rkvenc_dev *enc = platform_get_drvdata(pdev);
	2528	+ if (strstr(np->name, "ccu")) {
	2529	+ dev_info(dev, "remove ccu\n");
	2530	+ } else if (strstr(np->name, "core")) {
	2531	+ struct mpp_dev *mpp = dev_get_drvdata(dev);
	2532	+ struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
1388	2533
1389		- dev_info(dev, "remove device\n");
1390		- rkvenc2_free_rcbbuf(pdev, enc);
1391		- mpp_dev_remove(&enc->mpp);
1392		- rkvenc_procfs_remove(&enc->mpp);
	2534	+ dev_info(dev, "remove core\n");
	2535	+ if (enc->ccu) {
	2536	+ mutex_lock(&enc->ccu->lock);
	2537	+ list_del_init(&enc->core_link);
	2538	+ enc->ccu->core_num--;
	2539	+ mutex_unlock(&enc->ccu->lock);
	2540	+ }
	2541	+ rkvenc2_free_rcbbuf(pdev, enc);
	2542	+ mpp_dev_remove(&enc->mpp);
	2543	+ rkvenc_procfs_remove(&enc->mpp);
	2544	+ } else {
	2545	+ struct mpp_dev *mpp = dev_get_drvdata(dev);
	2546	+ struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
	2547	+
	2548	+ dev_info(dev, "remove device\n");
	2549	+ rkvenc2_free_rcbbuf(pdev, enc);
	2550	+ mpp_dev_remove(mpp);
	2551	+ rkvenc_procfs_remove(mpp);
	2552	+ }
1393	2553
1394	2554	return 0;
1395	2555	}
..	..	@@ -1397,23 +2557,9 @@
1397	2557	static void rkvenc_shutdown(struct platform_device *pdev)
1398	2558	{
1399	2559	struct device *dev = &pdev->dev;
1400		- int ret;
1401		- int val;
1402		- struct rkvenc_dev *enc = platform_get_drvdata(pdev);
1403		- struct mpp_dev *mpp = &enc->mpp;
1404	2560
1405		- dev_info(dev, "shutdown device\n");
1406		-
1407		- if (mpp->srv)
1408		- atomic_inc(&mpp->srv->shutdown_request);
1409		-
1410		- ret = readx_poll_timeout(atomic_read,
1411		- &mpp->task_count,
1412		- val, val == 0, 1000, 200000);
1413		- if (ret == -ETIMEDOUT)
1414		- dev_err(dev, "wait total running time out\n");
1415		-
1416		- dev_info(dev, "shutdown success\n");
	2561	+ if (!strstr(dev_name(dev), "ccu"))
	2562	+ mpp_dev_shutdown(pdev);
1417	2563	}
1418	2564
1419	2565	struct platform_driver rockchip_rkvenc2_driver = {