~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,13 +1,9 @@
	1	+// SPDX-License-Identifier: GPL-2.0+
1	2	/*
2	3	* V4L2 Capture CSI Subdev for Freescale i.MX5/6 SOC
3	4	*
4	5	* Copyright (c) 2014-2017 Mentor Graphics Inc.
5	6	* Copyright (C) 2017 Pengutronix, Philipp Zabel <kernel@pengutronix.de>
6		- *
7		- * This program is free software; you can redistribute it and/or modify
8		- * it under the terms of the GNU General Public License as published by
9		- * the Free Software Foundation; either version 2 of the License, or
10		- * (at your option) any later version.
11	7	*/
12	8	#include <linux/delay.h>
13	9	#include <linux/gcd.h>
..	..	@@ -37,11 +33,11 @@
37	33	* has not requested planar formats, we should allow 8 pixel
38	34	* alignment.
39	35	*/
40		-#define MIN_W 176
41		-#define MIN_H 144
	36	+#define MIN_W 32
	37	+#define MIN_H 32
42	38	#define MAX_W 4096
43	39	#define MAX_H 4096
44		-#define W_ALIGN 4 /* multiple of 16 pixels */
	40	+#define W_ALIGN 1 /* multiple of 2 pixels */
45	41	#define H_ALIGN 1 /* multiple of 2 lines */
46	42	#define S_ALIGN 1 /* multiple of 2 */
47	43
..	..	@@ -60,9 +56,10 @@
60	56	struct csi_priv {
61	57	struct device *dev;
62	58	struct ipu_soc *ipu;
63		- struct imx_media_dev *md;
64	59	struct v4l2_subdev sd;
65	60	struct media_pad pad[CSI_NUM_PADS];
	61	+ struct v4l2_async_notifier notifier;
	62	+
66	63	/* the video device at IDMAC output pad */
67	64	struct imx_media_video_dev *vdev;
68	65	struct imx_media_fim *fim;
..	..	@@ -114,6 +111,7 @@
114	111	u32 frame_sequence; /* frame sequence counter */
115	112	bool last_eof; /* waiting for last EOF at stream off */
116	113	bool nfb4eof; /* NFB4EOF encountered during streaming */
	114	+ bool interweave_swap; /* swap top/bottom lines when interweaving */
117	115	struct completion last_eof_comp;
118	116	};
119	117
..	..	@@ -122,9 +120,14 @@
122	120	return container_of(sdev, struct csi_priv, sd);
123	121	}
124	122
	123	+static inline struct csi_priv notifier_to_dev(struct v4l2_async_notifier n)
	124	+{
	125	+ return container_of(n, struct csi_priv, notifier);
	126	+}
	127	+
125	128	static inline bool is_parallel_bus(struct v4l2_fwnode_endpoint *ep)
126	129	{
127		- return ep->bus_type != V4L2_MBUS_CSI2;
	130	+ return ep->bus_type != V4L2_MBUS_CSI2_DPHY;
128	131	}
129	132
130	133	static inline bool is_parallel_16bit_bus(struct v4l2_fwnode_endpoint *ep)
..	..	@@ -161,8 +164,7 @@
161	164	static int csi_get_upstream_endpoint(struct csi_priv *priv,
162	165	struct v4l2_fwnode_endpoint *ep)
163	166	{
164		- struct device_node endpoint, port;
165		- struct media_entity *src;
	167	+ struct fwnode_handle *endpoint;
166	168	struct v4l2_subdev *sd;
167	169	struct media_pad *pad;
168	170
..	..	@@ -173,50 +175,43 @@
173	175	return -EPIPE;
174	176
175	177	sd = priv->src_sd;
176		- src = &sd->entity;
177	178
178		- if (src->function == MEDIA_ENT_F_VID_MUX) {
	179	+ switch (sd->grp_id) {
	180	+ case IMX_MEDIA_GRP_ID_CSI_MUX:
179	181	/*
180		- * CSI is connected directly to video mux, skip up to
	182	+ * CSI is connected directly to CSI mux, skip up to
181	183	* CSI-2 receiver if it is in the path, otherwise stay
182		- * with video mux.
	184	+ * with the CSI mux.
183	185	*/
184		- sd = imx_media_find_upstream_subdev(priv->md, src,
185		- IMX_MEDIA_GRP_ID_CSI2);
186		- if (!IS_ERR(sd))
187		- src = &sd->entity;
	186	+ sd = imx_media_pipeline_subdev(&sd->entity,
	187	+ IMX_MEDIA_GRP_ID_CSI2,
	188	+ true);
	189	+ if (IS_ERR(sd))
	190	+ sd = priv->src_sd;
	191	+ break;
	192	+ case IMX_MEDIA_GRP_ID_CSI2:
	193	+ break;
	194	+ default:
	195	+ /*
	196	+ * the source is neither the CSI mux nor the CSI-2 receiver,
	197	+ * get the source pad directly upstream from CSI itself.
	198	+ */
	199	+ sd = &priv->sd;
	200	+ break;
188	201	}
189	202
190		- /*
191		- * If the source is neither the video mux nor the CSI-2 receiver,
192		- * get the source pad directly upstream from CSI itself.
193		- */
194		- if (src->function != MEDIA_ENT_F_VID_MUX &&
195		- sd->grp_id != IMX_MEDIA_GRP_ID_CSI2)
196		- src = &priv->sd.entity;
197		-
198		- /* get source pad of entity directly upstream from src */
199		- pad = imx_media_find_upstream_pad(priv->md, src, 0);
200		- if (IS_ERR(pad))
201		- return PTR_ERR(pad);
202		-
203		- sd = media_entity_to_v4l2_subdev(pad->entity);
204		-
205		- /*
206		- * NOTE: this assumes an OF-graph port id is the same as a
207		- * media pad index.
208		- */
209		- port = of_graph_get_port_by_id(sd->dev->of_node, pad->index);
210		- if (!port)
	203	+ /* get source pad of entity directly upstream from sd */
	204	+ pad = imx_media_pipeline_pad(&sd->entity, 0, 0, true);
	205	+ if (!pad)
211	206	return -ENODEV;
212	207
213		- endpoint = of_get_next_child(port, NULL);
214		- of_node_put(port);
215		- if (!endpoint)
216		- return -ENODEV;
	208	+ endpoint = imx_media_get_pad_fwnode(pad);
	209	+ if (IS_ERR(endpoint))
	210	+ return PTR_ERR(endpoint);
217	211
218		- v4l2_fwnode_endpoint_parse(of_fwnode_handle(endpoint), ep);
219		- of_node_put(endpoint);
	212	+ v4l2_fwnode_endpoint_parse(endpoint, ep);
	213	+
	214	+ fwnode_handle_put(endpoint);
220	215
221	216	return 0;
222	217	}
..	..	@@ -295,6 +290,9 @@
295	290
296	291	if (ipu_idmac_buffer_is_ready(priv->idmac_ch, priv->ipu_buf_num))
297	292	ipu_idmac_clear_buffer(priv->idmac_ch, priv->ipu_buf_num);
	293	+
	294	+ if (priv->interweave_swap)
	295	+ phys += vdev->fmt.fmt.pix.bytesperline;
298	296
299	297	ipu_cpmem_set_buffer(priv->idmac_ch, priv->ipu_buf_num, phys);
300	298	}
..	..	@@ -408,23 +406,24 @@
408	406	struct imx_media_video_dev *vdev = priv->vdev;
409	407	const struct imx_media_pixfmt *incc;
410	408	struct v4l2_mbus_framefmt *infmt;
	409	+ struct v4l2_mbus_framefmt *outfmt;
	410	+ bool passthrough, interweave;
411	411	struct ipu_image image;
412	412	u32 passthrough_bits;
413	413	u32 passthrough_cycles;
414	414	dma_addr_t phys[2];
415		- bool passthrough;
416	415	u32 burst_size;
417	416	int ret;
418	417
419	418	infmt = &priv->format_mbus[CSI_SINK_PAD];
420	419	incc = priv->cc[CSI_SINK_PAD];
	420	+ outfmt = &priv->format_mbus[CSI_SRC_PAD_IDMAC];
421	421
422	422	ipu_cpmem_zero(priv->idmac_ch);
423	423
424	424	memset(&image, 0, sizeof(image));
425	425	image.pix = vdev->fmt.fmt.pix;
426		- image.rect.width = image.pix.width;
427		- image.rect.height = image.pix.height;
	426	+ image.rect = vdev->compose;
428	427
429	428	csi_idmac_setup_vb2_buf(priv, phys);
430	429
..	..	@@ -433,6 +432,16 @@
433	432
434	433	passthrough = requires_passthrough(&priv->upstream_ep, infmt, incc);
435	434	passthrough_cycles = 1;
	435	+
	436	+ /*
	437	+ * If the field type at capture interface is interlaced, and
	438	+ * the output IDMAC pad is sequential, enable interweave at
	439	+ * the IDMAC output channel.
	440	+ */
	441	+ interweave = V4L2_FIELD_IS_INTERLACED(image.pix.field) &&
	442	+ V4L2_FIELD_IS_SEQUENTIAL(outfmt->field);
	443	+ priv->interweave_swap = interweave &&
	444	+ image.pix.field == V4L2_FIELD_INTERLACED_BT;
436	445
437	446	switch (image.pix.pixelformat) {
438	447	case V4L2_PIX_FMT_SBGGR8:
..	..	@@ -447,18 +456,24 @@
447	456	case V4L2_PIX_FMT_SGBRG16:
448	457	case V4L2_PIX_FMT_SGRBG16:
449	458	case V4L2_PIX_FMT_SRGGB16:
450		- case V4L2_PIX_FMT_Y16:
	459	+ case V4L2_PIX_FMT_Y10:
	460	+ case V4L2_PIX_FMT_Y12:
451	461	burst_size = 8;
452	462	passthrough_bits = 16;
453	463	break;
454	464	case V4L2_PIX_FMT_YUV420:
	465	+ case V4L2_PIX_FMT_YVU420:
455	466	case V4L2_PIX_FMT_NV12:
456	467	burst_size = (image.pix.width & 0x3f) ?
457	468	((image.pix.width & 0x1f) ?
458	469	((image.pix.width & 0xf) ? 8 : 16) : 32) : 64;
459	470	passthrough_bits = 16;
460		- /* Skip writing U and V components to odd rows */
461		- ipu_cpmem_skip_odd_chroma_rows(priv->idmac_ch);
	471	+ /*
	472	+ * Skip writing U and V components to odd rows (but not
	473	+ * when enabling IDMAC interweaving, they are incompatible).
	474	+ */
	475	+ if (!interweave)
	476	+ ipu_cpmem_skip_odd_chroma_rows(priv->idmac_ch);
462	477	break;
463	478	case V4L2_PIX_FMT_YUYV:
464	479	case V4L2_PIX_FMT_UYVY:
..	..	@@ -473,7 +488,7 @@
473	488	passthrough_cycles = incc->cycles;
474	489	break;
475	490	}
476		- /* fallthrough - non-passthrough RGB565 (CSI-2 bus) */
	491	+ fallthrough; /* non-passthrough RGB565 (CSI-2 bus) */
477	492	default:
478	493	burst_size = (image.pix.width & 0xf) ? 8 : 16;
479	494	passthrough_bits = 16;
..	..	@@ -481,6 +496,12 @@
481	496	}
482	497
483	498	if (passthrough) {
	499	+ if (priv->interweave_swap) {
	500	+ /* start interweave scan at 1st top line (2nd line) */
	501	+ image.phys0 += image.pix.bytesperline;
	502	+ image.phys1 += image.pix.bytesperline;
	503	+ }
	504	+
484	505	ipu_cpmem_set_resolution(priv->idmac_ch,
485	506	image.rect.width * passthrough_cycles,
486	507	image.rect.height);
..	..	@@ -490,6 +511,11 @@
490	511	ipu_cpmem_set_format_passthrough(priv->idmac_ch,
491	512	passthrough_bits);
492	513	} else {
	514	+ if (priv->interweave_swap) {
	515	+ /* start interweave scan at 1st top line (2nd line) */
	516	+ image.rect.top = 1;
	517	+ }
	518	+
493	519	ret = ipu_cpmem_set_image(priv->idmac_ch, &image);
494	520	if (ret)
495	521	goto unsetup_vb2;
..	..	@@ -519,10 +545,12 @@
519	545
520	546	ipu_smfc_set_burstsize(priv->smfc, burst_size);
521	547
522		- if (image.pix.field == V4L2_FIELD_NONE &&
523		- V4L2_FIELD_HAS_BOTH(infmt->field))
	548	+ if (interweave)
524	549	ipu_cpmem_interlaced_scan(priv->idmac_ch,
525		- image.pix.bytesperline);
	550	+ priv->interweave_swap ?
	551	+ -image.pix.bytesperline :
	552	+ image.pix.bytesperline,
	553	+ image.pix.pixelformat);
526	554
527	555	ipu_idmac_set_double_buffer(priv->idmac_ch, true);
528	556
..	..	@@ -579,7 +607,7 @@
579	607
580	608	outfmt = &vdev->fmt.fmt.pix;
581	609
582		- ret = imx_media_alloc_dma_buf(priv->md, &priv->underrun_buf,
	610	+ ret = imx_media_alloc_dma_buf(priv->dev, &priv->underrun_buf,
583	611	outfmt->sizeimage);
584	612	if (ret)
585	613	goto out_put_ipu;
..	..	@@ -599,8 +627,8 @@
599	627	}
600	628
601	629	priv->nfb4eof_irq = ipu_idmac_channel_irq(priv->ipu,
602		- priv->idmac_ch,
603		- IPU_IRQ_NFB4EOF);
	630	+ priv->idmac_ch,
	631	+ IPU_IRQ_NFB4EOF);
604	632	ret = devm_request_irq(priv->dev, priv->nfb4eof_irq,
605	633	csi_idmac_nfb4eof_interrupt, 0,
606	634	"imx-smfc-nfb4eof", priv);
..	..	@@ -633,7 +661,7 @@
633	661	out_unsetup:
634	662	csi_idmac_unsetup(priv, VB2_BUF_STATE_QUEUED);
635	663	out_free_dma_buf:
636		- imx_media_free_dma_buf(priv->md, &priv->underrun_buf);
	664	+ imx_media_free_dma_buf(priv->dev, &priv->underrun_buf);
637	665	out_put_ipu:
638	666	csi_idmac_put_ipu_resources(priv);
639	667	return ret;
..	..	@@ -665,7 +693,7 @@
665	693
666	694	csi_idmac_unsetup(priv, VB2_BUF_STATE_ERROR);
667	695
668		- imx_media_free_dma_buf(priv->md, &priv->underrun_buf);
	696	+ imx_media_free_dma_buf(priv->dev, &priv->underrun_buf);
669	697
670	698	/* cancel the EOF timeout timer */
671	699	del_timer_sync(&priv->eof_timeout_timer);
..	..	@@ -692,12 +720,7 @@
692	720	priv->upstream_ep.bus.parallel.flags :
693	721	priv->upstream_ep.bus.mipi_csi2.flags;
694	722
695		- /*
696		- * we need to pass input frame to CSI interface, but
697		- * with translated field type from output format
698		- */
699	723	if_fmt = *infmt;
700		- if_fmt.field = outfmt->field;
701	724	crop = priv->crop;
702	725
703	726	/*
..	..	@@ -715,7 +738,7 @@
715	738	priv->crop.width == 2 * priv->compose.width,
716	739	priv->crop.height == 2 * priv->compose.height);
717	740
718		- ipu_csi_init_interface(priv->csi, &mbus_cfg, &if_fmt);
	741	+ ipu_csi_init_interface(priv->csi, &mbus_cfg, &if_fmt, outfmt);
719	742
720	743	ipu_csi_set_dest(priv->csi, priv->dest);
721	744
..	..	@@ -920,7 +943,10 @@
920	943
921	944	switch (fi->pad) {
922	945	case CSI_SINK_PAD:
923		- /* No limits on input frame interval */
	946	+ /* No limits on valid input frame intervals */
	947	+ if (fi->interval.numerator == 0 \|\|
	948	+ fi->interval.denominator == 0)
	949	+ fi->interval = *input_fi;
924	950	/* Reset output intervals and frame skipping ratio to 1:1 */
925	951	priv->frame_interval[CSI_SRC_PAD_IDMAC] = fi->interval;
926	952	priv->frame_interval[CSI_SRC_PAD_DIRECT] = fi->interval;
..	..	@@ -1035,6 +1061,8 @@
1035	1061	v4l2_ctrl_handler_free(&priv->ctrl_hdlr);
1036	1062	v4l2_ctrl_handler_init(&priv->ctrl_hdlr, 0);
1037	1063	priv->sink = NULL;
	1064	+ /* do not apply IC burst alignment in csi_try_crop */
	1065	+ priv->active_output_pad = CSI_SRC_PAD_IDMAC;
1038	1066	goto out;
1039	1067	}
1040	1068
..	..	@@ -1063,10 +1091,10 @@
1063	1091
1064	1092	remote_sd = media_entity_to_v4l2_subdev(remote->entity);
1065	1093	switch (remote_sd->grp_id) {
1066		- case IMX_MEDIA_GRP_ID_VDIC:
	1094	+ case IMX_MEDIA_GRP_ID_IPU_VDIC:
1067	1095	priv->dest = IPU_CSI_DEST_VDIC;
1068	1096	break;
1069		- case IMX_MEDIA_GRP_ID_IC_PRP:
	1097	+ case IMX_MEDIA_GRP_ID_IPU_IC_PRP:
1070	1098	priv->dest = IPU_CSI_DEST_IC;
1071	1099	break;
1072	1100	default:
..	..	@@ -1087,7 +1115,7 @@
1087	1115	struct v4l2_subdev_format *sink_fmt)
1088	1116	{
1089	1117	struct csi_priv *priv = v4l2_get_subdevdata(sd);
1090		- struct v4l2_fwnode_endpoint upstream_ep;
	1118	+ struct v4l2_fwnode_endpoint upstream_ep = { .bus_type = 0 };
1091	1119	bool is_csi2;
1092	1120	int ret;
1093	1121
..	..	@@ -1117,8 +1145,7 @@
1117	1145	*/
1118	1146	#if 0
1119	1147	mutex_unlock(&priv->lock);
1120		- vc_num = imx_media_find_mipi_csi2_channel(priv->md,
1121		- &priv->sd.entity);
	1148	+ vc_num = imx_media_find_mipi_csi2_channel(&priv->sd.entity);
1122	1149	if (vc_num < 0)
1123	1150	return vc_num;
1124	1151	mutex_lock(&priv->lock);
..	..	@@ -1171,12 +1198,21 @@
1171	1198	struct v4l2_mbus_framefmt *infmt,
1172	1199	struct v4l2_fwnode_endpoint *upstream_ep)
1173	1200	{
	1201	+ u32 in_height;
	1202	+
1174	1203	crop->width = min_t(__u32, infmt->width, crop->width);
1175	1204	if (crop->left + crop->width > infmt->width)
1176	1205	crop->left = infmt->width - crop->width;
1177	1206	/* adjust crop left/width to h/w alignment restrictions */
1178	1207	crop->left &= ~0x3;
1179		- crop->width &= ~0x7;
	1208	+ if (priv->active_output_pad == CSI_SRC_PAD_DIRECT)
	1209	+ crop->width &= ~0x7; /* multiple of 8 pixels (IC burst) */
	1210	+ else
	1211	+ crop->width &= ~0x1; /* multiple of 2 pixels */
	1212	+
	1213	+ in_height = infmt->height;
	1214	+ if (infmt->field == V4L2_FIELD_ALTERNATE)
	1215	+ in_height *= 2;
1180	1216
1181	1217	/*
1182	1218	* FIXME: not sure why yet, but on interlaced bt.656,
..	..	@@ -1187,12 +1223,12 @@
1187	1223	if (upstream_ep->bus_type == V4L2_MBUS_BT656 &&
1188	1224	(V4L2_FIELD_HAS_BOTH(infmt->field) \|\|
1189	1225	infmt->field == V4L2_FIELD_ALTERNATE)) {
1190		- crop->height = infmt->height;
1191		- crop->top = (infmt->height == 480) ? 2 : 0;
	1226	+ crop->height = in_height;
	1227	+ crop->top = (in_height == 480) ? 2 : 0;
1192	1228	} else {
1193		- crop->height = min_t(__u32, infmt->height, crop->height);
1194		- if (crop->top + crop->height > infmt->height)
1195		- crop->top = infmt->height - crop->height;
	1229	+ crop->height = min_t(__u32, in_height, crop->height);
	1230	+ if (crop->top + crop->height > in_height)
	1231	+ crop->top = in_height - crop->height;
1196	1232	}
1197	1233	}
1198	1234
..	..	@@ -1201,7 +1237,7 @@
1201	1237	struct v4l2_subdev_mbus_code_enum *code)
1202	1238	{
1203	1239	struct csi_priv *priv = v4l2_get_subdevdata(sd);
1204		- struct v4l2_fwnode_endpoint upstream_ep;
	1240	+ struct v4l2_fwnode_endpoint upstream_ep = { .bus_type = 0 };
1205	1241	const struct imx_media_pixfmt *incc;
1206	1242	struct v4l2_mbus_framefmt *infmt;
1207	1243	int ret = 0;
..	..	@@ -1209,12 +1245,12 @@
1209	1245	mutex_lock(&priv->lock);
1210	1246
1211	1247	infmt = __csi_get_fmt(priv, cfg, CSI_SINK_PAD, code->which);
1212		- incc = imx_media_find_mbus_format(infmt->code, CS_SEL_ANY, true);
	1248	+ incc = imx_media_find_mbus_format(infmt->code, PIXFMT_SEL_ANY);
1213	1249
1214	1250	switch (code->pad) {
1215	1251	case CSI_SINK_PAD:
1216		- ret = imx_media_enum_mbus_format(&code->code, code->index,
1217		- CS_SEL_ANY, true);
	1252	+ ret = imx_media_enum_mbus_formats(&code->code, code->index,
	1253	+ PIXFMT_SEL_ANY);
1218	1254	break;
1219	1255	case CSI_SRC_PAD_DIRECT:
1220	1256	case CSI_SRC_PAD_IDMAC:
..	..	@@ -1231,11 +1267,13 @@
1231	1267	}
1232	1268	code->code = infmt->code;
1233	1269	} else {
1234		- u32 cs_sel = (incc->cs == IPUV3_COLORSPACE_YUV) ?
1235		- CS_SEL_YUV : CS_SEL_RGB;
1236		- ret = imx_media_enum_ipu_format(&code->code,
1237		- code->index,
1238		- cs_sel);
	1270	+ enum imx_pixfmt_sel fmt_sel =
	1271	+ (incc->cs == IPUV3_COLORSPACE_YUV) ?
	1272	+ PIXFMT_SEL_YUV : PIXFMT_SEL_RGB;
	1273	+
	1274	+ ret = imx_media_enum_ipu_formats(&code->code,
	1275	+ code->index,
	1276	+ fmt_sel);
1239	1277	}
1240	1278	break;
1241	1279	default:
..	..	@@ -1342,6 +1380,55 @@
1342	1380	return ret;
1343	1381	}
1344	1382
	1383	+static void csi_try_field(struct csi_priv *priv,
	1384	+ struct v4l2_subdev_pad_config *cfg,
	1385	+ struct v4l2_subdev_format *sdformat)
	1386	+{
	1387	+ struct v4l2_mbus_framefmt *infmt =
	1388	+ __csi_get_fmt(priv, cfg, CSI_SINK_PAD, sdformat->which);
	1389	+
	1390	+ /*
	1391	+ * no restrictions on sink pad field type except must
	1392	+ * be initialized.
	1393	+ */
	1394	+ if (sdformat->pad == CSI_SINK_PAD) {
	1395	+ if (sdformat->format.field == V4L2_FIELD_ANY)
	1396	+ sdformat->format.field = V4L2_FIELD_NONE;
	1397	+ return;
	1398	+ }
	1399	+
	1400	+ switch (infmt->field) {
	1401	+ case V4L2_FIELD_SEQ_TB:
	1402	+ case V4L2_FIELD_SEQ_BT:
	1403	+ /*
	1404	+ * If the user requests sequential at the source pad,
	1405	+ * allow it (along with possibly inverting field order).
	1406	+ * Otherwise passthrough the field type.
	1407	+ */
	1408	+ if (!V4L2_FIELD_IS_SEQUENTIAL(sdformat->format.field))
	1409	+ sdformat->format.field = infmt->field;
	1410	+ break;
	1411	+ case V4L2_FIELD_ALTERNATE:
	1412	+ /*
	1413	+ * This driver does not support alternate field mode, and
	1414	+ * the CSI captures a whole frame, so the CSI never presents
	1415	+ * alternate mode at its source pads. If user has not
	1416	+ * already requested sequential, translate ALTERNATE at
	1417	+ * sink pad to SEQ_TB or SEQ_BT at the source pad depending
	1418	+ * on input height (assume NTSC BT order if 480 total active
	1419	+ * frame lines, otherwise PAL TB order).
	1420	+ */
	1421	+ if (!V4L2_FIELD_IS_SEQUENTIAL(sdformat->format.field))
	1422	+ sdformat->format.field = (infmt->height == 480 / 2) ?
	1423	+ V4L2_FIELD_SEQ_BT : V4L2_FIELD_SEQ_TB;
	1424	+ break;
	1425	+ default:
	1426	+ /* Passthrough for all other input field types */
	1427	+ sdformat->format.field = infmt->field;
	1428	+ break;
	1429	+ }
	1430	+}
	1431	+
1345	1432	static void csi_try_fmt(struct csi_priv *priv,
1346	1433	struct v4l2_fwnode_endpoint *upstream_ep,
1347	1434	struct v4l2_subdev_pad_config *cfg,
..	..	@@ -1359,8 +1446,7 @@
1359	1446	switch (sdformat->pad) {
1360	1447	case CSI_SRC_PAD_DIRECT:
1361	1448	case CSI_SRC_PAD_IDMAC:
1362		- incc = imx_media_find_mbus_format(infmt->code,
1363		- CS_SEL_ANY, true);
	1449	+ incc = imx_media_find_mbus_format(infmt->code, PIXFMT_SEL_ANY);
1364	1450
1365	1451	sdformat->format.width = compose->width;
1366	1452	sdformat->format.height = compose->height;
..	..	@@ -1369,69 +1455,63 @@
1369	1455	sdformat->format.code = infmt->code;
1370	1456	*cc = incc;
1371	1457	} else {
1372		- u32 cs_sel = (incc->cs == IPUV3_COLORSPACE_YUV) ?
1373		- CS_SEL_YUV : CS_SEL_RGB;
	1458	+ enum imx_pixfmt_sel fmt_sel =
	1459	+ (incc->cs == IPUV3_COLORSPACE_YUV) ?
	1460	+ PIXFMT_SEL_YUV : PIXFMT_SEL_RGB;
1374	1461
1375	1462	*cc = imx_media_find_ipu_format(sdformat->format.code,
1376		- cs_sel);
	1463	+ fmt_sel);
1377	1464	if (!*cc) {
1378		- imx_media_enum_ipu_format(&code, 0, cs_sel);
1379		- *cc = imx_media_find_ipu_format(code, cs_sel);
	1465	+ imx_media_enum_ipu_formats(&code, 0, fmt_sel);
	1466	+ *cc = imx_media_find_ipu_format(code, fmt_sel);
1380	1467	sdformat->format.code = (*cc)->codes[0];
1381	1468	}
1382	1469	}
1383	1470
1384		- if (sdformat->pad == CSI_SRC_PAD_DIRECT \|\|
1385		- sdformat->format.field != V4L2_FIELD_NONE)
1386		- sdformat->format.field = infmt->field;
1387		-
1388		- /*
1389		- * translate V4L2_FIELD_ALTERNATE to SEQ_TB or SEQ_BT
1390		- * depending on input height (assume NTSC top-bottom
1391		- * order if 480 lines, otherwise PAL bottom-top order).
1392		- */
1393		- if (sdformat->format.field == V4L2_FIELD_ALTERNATE) {
1394		- sdformat->format.field = (infmt->height == 480) ?
1395		- V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT;
1396		- }
	1471	+ csi_try_field(priv, cfg, sdformat);
1397	1472
1398	1473	/* propagate colorimetry from sink */
1399	1474	sdformat->format.colorspace = infmt->colorspace;
1400	1475	sdformat->format.xfer_func = infmt->xfer_func;
1401	1476	sdformat->format.quantization = infmt->quantization;
1402	1477	sdformat->format.ycbcr_enc = infmt->ycbcr_enc;
	1478	+
1403	1479	break;
1404	1480	case CSI_SINK_PAD:
1405	1481	v4l_bound_align_image(&sdformat->format.width, MIN_W, MAX_W,
1406	1482	W_ALIGN, &sdformat->format.height,
1407	1483	MIN_H, MAX_H, H_ALIGN, S_ALIGN);
1408	1484
	1485	+ *cc = imx_media_find_mbus_format(sdformat->format.code,
	1486	+ PIXFMT_SEL_ANY);
	1487	+ if (!*cc) {
	1488	+ imx_media_enum_mbus_formats(&code, 0,
	1489	+ PIXFMT_SEL_YUV_RGB);
	1490	+ *cc = imx_media_find_mbus_format(code,
	1491	+ PIXFMT_SEL_YUV_RGB);
	1492	+ sdformat->format.code = (*cc)->codes[0];
	1493	+ }
	1494	+
	1495	+ csi_try_field(priv, cfg, sdformat);
	1496	+
1409	1497	/* Reset crop and compose rectangles */
1410	1498	crop->left = 0;
1411	1499	crop->top = 0;
1412	1500	crop->width = sdformat->format.width;
1413	1501	crop->height = sdformat->format.height;
	1502	+ if (sdformat->format.field == V4L2_FIELD_ALTERNATE)
	1503	+ crop->height *= 2;
1414	1504	csi_try_crop(priv, crop, cfg, &sdformat->format, upstream_ep);
1415	1505	compose->left = 0;
1416	1506	compose->top = 0;
1417	1507	compose->width = crop->width;
1418	1508	compose->height = crop->height;
1419	1509
1420		- *cc = imx_media_find_mbus_format(sdformat->format.code,
1421		- CS_SEL_ANY, true);
1422		- if (!*cc) {
1423		- imx_media_enum_mbus_format(&code, 0,
1424		- CS_SEL_ANY, false);
1425		- *cc = imx_media_find_mbus_format(code,
1426		- CS_SEL_ANY, false);
1427		- sdformat->format.code = (*cc)->codes[0];
1428		- }
1429		-
1430		- imx_media_fill_default_mbus_fields(
1431		- &sdformat->format, infmt,
1432		- priv->active_output_pad == CSI_SRC_PAD_DIRECT);
1433	1510	break;
1434	1511	}
	1512	+
	1513	+ imx_media_try_colorimetry(&sdformat->format,
	1514	+ priv->active_output_pad == CSI_SRC_PAD_DIRECT);
1435	1515	}
1436	1516
1437	1517	static int csi_set_fmt(struct v4l2_subdev *sd,
..	..	@@ -1439,10 +1519,8 @@
1439	1519	struct v4l2_subdev_format *sdformat)
1440	1520	{
1441	1521	struct csi_priv *priv = v4l2_get_subdevdata(sd);
1442		- struct imx_media_video_dev *vdev = priv->vdev;
1443		- struct v4l2_fwnode_endpoint upstream_ep;
	1522	+ struct v4l2_fwnode_endpoint upstream_ep = { .bus_type = 0 };
1444	1523	const struct imx_media_pixfmt *cc;
1445		- struct v4l2_pix_format vdev_fmt;
1446	1524	struct v4l2_mbus_framefmt *fmt;
1447	1525	struct v4l2_rect crop, compose;
1448	1526	int ret;
..	..	@@ -1494,19 +1572,9 @@
1494	1572	}
1495	1573	}
1496	1574
1497		- if (sdformat->which == V4L2_SUBDEV_FORMAT_TRY)
1498		- goto out;
	1575	+ if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE)
	1576	+ priv->cc[sdformat->pad] = cc;
1499	1577
1500		- priv->cc[sdformat->pad] = cc;
1501		-
1502		- /* propagate IDMAC output pad format to capture device */
1503		- imx_media_mbus_fmt_to_pix_fmt(&vdev_fmt,
1504		- &priv->format_mbus[CSI_SRC_PAD_IDMAC],
1505		- priv->cc[CSI_SRC_PAD_IDMAC]);
1506		- mutex_unlock(&priv->lock);
1507		- imx_media_capture_device_set_format(vdev, &vdev_fmt);
1508		-
1509		- return 0;
1510	1578	out:
1511	1579	mutex_unlock(&priv->lock);
1512	1580	return ret;
..	..	@@ -1536,6 +1604,8 @@
1536	1604	sel->r.top = 0;
1537	1605	sel->r.width = infmt->width;
1538	1606	sel->r.height = infmt->height;
	1607	+ if (infmt->field == V4L2_FIELD_ALTERNATE)
	1608	+ sel->r.height *= 2;
1539	1609	break;
1540	1610	case V4L2_SEL_TGT_CROP:
1541	1611	sel->r = *crop;
..	..	@@ -1579,7 +1649,7 @@
1579	1649	struct v4l2_subdev_selection *sel)
1580	1650	{
1581	1651	struct csi_priv *priv = v4l2_get_subdevdata(sd);
1582		- struct v4l2_fwnode_endpoint upstream_ep;
	1652	+ struct v4l2_fwnode_endpoint upstream_ep = { .bus_type = 0 };
1583	1653	struct v4l2_mbus_framefmt *infmt;
1584	1654	struct v4l2_rect crop, compose;
1585	1655	int pad, ret;
..	..	@@ -1686,18 +1756,12 @@
1686	1756	return v4l2_event_unsubscribe(fh, sub);
1687	1757	}
1688	1758
1689		-/*
1690		- * retrieve our pads parsed from the OF graph by the media device
1691		- */
1692	1759	static int csi_registered(struct v4l2_subdev *sd)
1693	1760	{
1694	1761	struct csi_priv *priv = v4l2_get_subdevdata(sd);
1695	1762	struct ipu_csi *csi;
1696	1763	int i, ret;
1697	1764	u32 code;
1698		-
1699		- /* get media device */
1700		- priv->md = dev_get_drvdata(sd->v4l2_dev->dev);
1701	1765
1702	1766	/* get handle to IPU CSI */
1703	1767	csi = ipu_csi_get(priv->ipu, priv->csi_id);
..	..	@@ -1708,12 +1772,9 @@
1708	1772	priv->csi = csi;
1709	1773
1710	1774	for (i = 0; i < CSI_NUM_PADS; i++) {
1711		- priv->pad[i].flags = (i == CSI_SINK_PAD) ?
1712		- MEDIA_PAD_FL_SINK : MEDIA_PAD_FL_SOURCE;
1713		-
1714	1775	code = 0;
1715	1776	if (i != CSI_SINK_PAD)
1716		- imx_media_enum_ipu_format(&code, 0, CS_SEL_YUV);
	1777	+ imx_media_enum_ipu_formats(&code, 0, PIXFMT_SEL_YUV);
1717	1778
1718	1779	/* set a default mbus format */
1719	1780	ret = imx_media_init_mbus_fmt(&priv->format_mbus[i],
..	..	@@ -1742,21 +1803,22 @@
1742	1803	goto put_csi;
1743	1804	}
1744	1805
1745		- ret = media_entity_pads_init(&sd->entity, CSI_NUM_PADS, priv->pad);
1746		- if (ret)
	1806	+ priv->vdev = imx_media_capture_device_init(priv->sd.dev,
	1807	+ &priv->sd,
	1808	+ CSI_SRC_PAD_IDMAC);
	1809	+ if (IS_ERR(priv->vdev)) {
	1810	+ ret = PTR_ERR(priv->vdev);
1747	1811	goto free_fim;
	1812	+ }
1748	1813
1749	1814	ret = imx_media_capture_device_register(priv->vdev);
1750	1815	if (ret)
1751		- goto free_fim;
1752		-
1753		- ret = imx_media_add_video_device(priv->md, priv->vdev);
1754		- if (ret)
1755		- goto unreg;
	1816	+ goto remove_vdev;
1756	1817
1757	1818	return 0;
1758		-unreg:
1759		- imx_media_capture_device_unregister(priv->vdev);
	1819	+
	1820	+remove_vdev:
	1821	+ imx_media_capture_device_remove(priv->vdev);
1760	1822	free_fim:
1761	1823	if (priv->fim)
1762	1824	imx_media_fim_free(priv->fim);
..	..	@@ -1770,6 +1832,7 @@
1770	1832	struct csi_priv *priv = v4l2_get_subdevdata(sd);
1771	1833
1772	1834	imx_media_capture_device_unregister(priv->vdev);
	1835	+ imx_media_capture_device_remove(priv->vdev);
1773	1836
1774	1837	if (priv->fim)
1775	1838	imx_media_fim_free(priv->fim);
..	..	@@ -1778,9 +1841,32 @@
1778	1841	ipu_csi_put(priv->csi);
1779	1842	}
1780	1843
	1844	+/*
	1845	+ * The CSI has only one fwnode endpoint, at the sink pad. Verify the
	1846	+ * endpoint belongs to us, and return CSI_SINK_PAD.
	1847	+ */
	1848	+static int csi_get_fwnode_pad(struct media_entity *entity,
	1849	+ struct fwnode_endpoint *endpoint)
	1850	+{
	1851	+ struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(entity);
	1852	+ struct csi_priv *priv = v4l2_get_subdevdata(sd);
	1853	+ struct fwnode_handle *csi_port = dev_fwnode(priv->dev);
	1854	+ struct fwnode_handle *csi_ep;
	1855	+ int ret;
	1856	+
	1857	+ csi_ep = fwnode_get_next_child_node(csi_port, NULL);
	1858	+
	1859	+ ret = endpoint->local_fwnode == csi_ep ? CSI_SINK_PAD : -ENXIO;
	1860	+
	1861	+ fwnode_handle_put(csi_ep);
	1862	+
	1863	+ return ret;
	1864	+}
	1865	+
1781	1866	static const struct media_entity_operations csi_entity_ops = {
1782	1867	.link_setup = csi_link_setup,
1783	1868	.link_validate = v4l2_subdev_link_validate,
	1869	+ .get_fwnode_pad = csi_get_fwnode_pad,
1784	1870	};
1785	1871
1786	1872	static const struct v4l2_subdev_core_ops csi_core_ops = {
..	..	@@ -1817,12 +1903,74 @@
1817	1903	.unregistered = csi_unregistered,
1818	1904	};
1819	1905
	1906	+static int imx_csi_notify_bound(struct v4l2_async_notifier *notifier,
	1907	+ struct v4l2_subdev *sd,
	1908	+ struct v4l2_async_subdev *asd)
	1909	+{
	1910	+ struct csi_priv *priv = notifier_to_dev(notifier);
	1911	+ struct media_pad *sink = &priv->sd.entity.pads[CSI_SINK_PAD];
	1912	+
	1913	+ /*
	1914	+ * If the subdev is a video mux, it must be one of the CSI
	1915	+ * muxes. Mark it as such via its group id.
	1916	+ */
	1917	+ if (sd->entity.function == MEDIA_ENT_F_VID_MUX)
	1918	+ sd->grp_id = IMX_MEDIA_GRP_ID_CSI_MUX;
	1919	+
	1920	+ return v4l2_create_fwnode_links_to_pad(sd, sink);
	1921	+}
	1922	+
	1923	+static const struct v4l2_async_notifier_operations csi_notify_ops = {
	1924	+ .bound = imx_csi_notify_bound,
	1925	+};
	1926	+
	1927	+static int imx_csi_async_register(struct csi_priv *priv)
	1928	+{
	1929	+ struct v4l2_async_subdev *asd = NULL;
	1930	+ struct fwnode_handle *ep;
	1931	+ unsigned int port;
	1932	+ int ret;
	1933	+
	1934	+ v4l2_async_notifier_init(&priv->notifier);
	1935	+
	1936	+ /* get this CSI's port id */
	1937	+ ret = fwnode_property_read_u32(dev_fwnode(priv->dev), "reg", &port);
	1938	+ if (ret < 0)
	1939	+ return ret;
	1940	+
	1941	+ ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(priv->dev->parent),
	1942	+ port, 0,
	1943	+ FWNODE_GRAPH_ENDPOINT_NEXT);
	1944	+ if (ep) {
	1945	+ asd = v4l2_async_notifier_add_fwnode_remote_subdev(
	1946	+ &priv->notifier, ep, sizeof(*asd));
	1947	+
	1948	+ fwnode_handle_put(ep);
	1949	+
	1950	+ if (IS_ERR(asd)) {
	1951	+ ret = PTR_ERR(asd);
	1952	+ /* OK if asd already exists */
	1953	+ if (ret != -EEXIST)
	1954	+ return ret;
	1955	+ }
	1956	+ }
	1957	+
	1958	+ priv->notifier.ops = &csi_notify_ops;
	1959	+
	1960	+ ret = v4l2_async_subdev_notifier_register(&priv->sd,
	1961	+ &priv->notifier);
	1962	+ if (ret)
	1963	+ return ret;
	1964	+
	1965	+ return v4l2_async_register_subdev(&priv->sd);
	1966	+}
	1967	+
1820	1968	static int imx_csi_probe(struct platform_device *pdev)
1821	1969	{
1822	1970	struct ipu_client_platformdata *pdata;
1823	1971	struct pinctrl *pinctrl;
1824	1972	struct csi_priv *priv;
1825		- int ret;
	1973	+ int i, ret;
1826	1974
1827	1975	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1828	1976	if (!priv)
..	..	@@ -1843,6 +1991,8 @@
1843	1991	priv->csi_id = pdata->csi;
1844	1992	priv->smfc_id = (priv->csi_id == 0) ? 0 : 2;
1845	1993
	1994	+ priv->active_output_pad = CSI_SRC_PAD_IDMAC;
	1995	+
1846	1996	timer_setup(&priv->eof_timeout_timer, csi_idmac_eof_timeout, 0);
1847	1997	spin_lock_init(&priv->irqlock);
1848	1998
..	..	@@ -1856,14 +2006,18 @@
1856	2006	priv->sd.owner = THIS_MODULE;
1857	2007	priv->sd.flags = V4L2_SUBDEV_FL_HAS_DEVNODE \| V4L2_SUBDEV_FL_HAS_EVENTS;
1858	2008	priv->sd.grp_id = priv->csi_id ?
1859		- IMX_MEDIA_GRP_ID_CSI1 : IMX_MEDIA_GRP_ID_CSI0;
	2009	+ IMX_MEDIA_GRP_ID_IPU_CSI1 : IMX_MEDIA_GRP_ID_IPU_CSI0;
1860	2010	imx_media_grp_id_to_sd_name(priv->sd.name, sizeof(priv->sd.name),
1861	2011	priv->sd.grp_id, ipu_get_num(priv->ipu));
1862	2012
1863		- priv->vdev = imx_media_capture_device_init(&priv->sd,
1864		- CSI_SRC_PAD_IDMAC);
1865		- if (IS_ERR(priv->vdev))
1866		- return PTR_ERR(priv->vdev);
	2013	+ for (i = 0; i < CSI_NUM_PADS; i++)
	2014	+ priv->pad[i].flags = (i == CSI_SINK_PAD) ?
	2015	+ MEDIA_PAD_FL_SINK : MEDIA_PAD_FL_SOURCE;
	2016	+
	2017	+ ret = media_entity_pads_init(&priv->sd.entity, CSI_NUM_PADS,
	2018	+ priv->pad);
	2019	+ if (ret)
	2020	+ return ret;
1867	2021
1868	2022	mutex_init(&priv->lock);
1869	2023
..	..	@@ -1886,15 +2040,18 @@
1886	2040	goto free;
1887	2041	}
1888	2042
1889		- ret = v4l2_async_register_subdev(&priv->sd);
	2043	+ ret = imx_csi_async_register(priv);
1890	2044	if (ret)
1891		- goto free;
	2045	+ goto cleanup;
1892	2046
1893	2047	return 0;
	2048	+
	2049	+cleanup:
	2050	+ v4l2_async_notifier_unregister(&priv->notifier);
	2051	+ v4l2_async_notifier_cleanup(&priv->notifier);
1894	2052	free:
1895	2053	v4l2_ctrl_handler_free(&priv->ctrl_hdlr);
1896	2054	mutex_destroy(&priv->lock);
1897		- imx_media_capture_device_remove(priv->vdev);
1898	2055	return ret;
1899	2056	}
1900	2057
..	..	@@ -1905,7 +2062,8 @@
1905	2062
1906	2063	v4l2_ctrl_handler_free(&priv->ctrl_hdlr);
1907	2064	mutex_destroy(&priv->lock);
1908		- imx_media_capture_device_remove(priv->vdev);
	2065	+ v4l2_async_notifier_unregister(&priv->notifier);
	2066	+ v4l2_async_notifier_cleanup(&priv->notifier);
1909	2067	v4l2_async_unregister_subdev(sd);
1910	2068	media_entity_cleanup(&sd->entity);
1911	2069