vsp1_wpf.c

  1// SPDX-License-Identifier: GPL-2.0+
  2/*
  3 * vsp1_wpf.c  --  R-Car VSP1 Write Pixel Formatter
  4 *
  5 * Copyright (C) 2013-2014 Renesas Electronics Corporation
  6 *
  7 * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
  8 */
  9
 10#include <linux/device.h>
 11
 12#include <media/v4l2-subdev.h>
 13
 14#include "vsp1.h"
 15#include "vsp1_dl.h"
 16#include "vsp1_pipe.h"
 17#include "vsp1_rwpf.h"
 18#include "vsp1_video.h"
 19
 20#define WPF_GEN2_MAX_WIDTH			2048U
 21#define WPF_GEN2_MAX_HEIGHT			2048U
 22#define WPF_GEN3_MAX_WIDTH			8190U
 23#define WPF_GEN3_MAX_HEIGHT			8190U
 24
 25/* -----------------------------------------------------------------------------
 26 * Device Access
 27 */
 28
 29static inline void vsp1_wpf_write(struct vsp1_rwpf *wpf,
 30				  struct vsp1_dl_body *dlb, u32 reg, u32 data)
 31{
 32	vsp1_dl_body_write(dlb, reg + wpf->entity.index * VI6_WPF_OFFSET, data);
 33}
 34
 35/* -----------------------------------------------------------------------------
 36 * Controls
 37 */
 38
 39enum wpf_flip_ctrl {
 40	WPF_CTRL_VFLIP = 0,
 41	WPF_CTRL_HFLIP = 1,
 42};
 43
 44static int vsp1_wpf_set_rotation(struct vsp1_rwpf *wpf, unsigned int rotation)
 45{
 46	struct vsp1_video *video = wpf->video;
 47	struct v4l2_mbus_framefmt *sink_format;
 48	struct v4l2_mbus_framefmt *source_format;
 49	bool rotate;
 50	int ret = 0;
 51
 52	/*
 53	 * Only consider the 0°/180° from/to 90°/270° modifications, the rest
 54	 * is taken care of by the flipping configuration.
 55	 */
 56	rotate = rotation == 90 || rotation == 270;
 57	if (rotate == wpf->flip.rotate)
 58		return 0;
 59
 60	/* Changing rotation isn't allowed when buffers are allocated. */
 61	mutex_lock(&video->lock);
 62
 63	if (vb2_is_busy(&video->queue)) {
 64		ret = -EBUSY;
 65		goto done;
 66	}
 67
 68	sink_format = v4l2_subdev_state_get_format(wpf->entity.state,
 69						   RWPF_PAD_SINK);
 70	source_format = v4l2_subdev_state_get_format(wpf->entity.state,
 71						     RWPF_PAD_SOURCE);
 72
 73	mutex_lock(&wpf->entity.lock);
 74
 75	if (rotate) {
 76		source_format->width = sink_format->height;
 77		source_format->height = sink_format->width;
 78	} else {
 79		source_format->width = sink_format->width;
 80		source_format->height = sink_format->height;
 81	}
 82
 83	wpf->flip.rotate = rotate;
 84
 85	mutex_unlock(&wpf->entity.lock);
 86
 87done:
 88	mutex_unlock(&video->lock);
 89	return ret;
 90}
 91
 92static int vsp1_wpf_s_ctrl(struct v4l2_ctrl *ctrl)
 93{
 94	struct vsp1_rwpf *wpf =
 95		container_of(ctrl->handler, struct vsp1_rwpf, ctrls);
 96	unsigned int rotation;
 97	u32 flip = 0;
 98	int ret;
 99
100	/* Update the rotation. */
101	rotation = wpf->flip.ctrls.rotate ? wpf->flip.ctrls.rotate->val : 0;
102	ret = vsp1_wpf_set_rotation(wpf, rotation);
103	if (ret < 0)
104		return ret;
105
106	/*
107	 * Compute the flip value resulting from all three controls, with
108	 * rotation by 180° flipping the image in both directions. Store the
109	 * result in the pending flip field for the next frame that will be
110	 * processed.
111	 */
112	if (wpf->flip.ctrls.vflip->val)
113		flip |= BIT(WPF_CTRL_VFLIP);
114
115	if (wpf->flip.ctrls.hflip && wpf->flip.ctrls.hflip->val)
116		flip |= BIT(WPF_CTRL_HFLIP);
117
118	if (rotation == 180 || rotation == 270)
119		flip ^= BIT(WPF_CTRL_VFLIP) | BIT(WPF_CTRL_HFLIP);
120
121	spin_lock_irq(&wpf->flip.lock);
122	wpf->flip.pending = flip;
123	spin_unlock_irq(&wpf->flip.lock);
124
125	return 0;
126}
127
128static const struct v4l2_ctrl_ops vsp1_wpf_ctrl_ops = {
129	.s_ctrl = vsp1_wpf_s_ctrl,
130};
131
132static int wpf_init_controls(struct vsp1_rwpf *wpf)
133{
134	struct vsp1_device *vsp1 = wpf->entity.vsp1;
135	unsigned int num_flip_ctrls;
136	int ret;
137
138	spin_lock_init(&wpf->flip.lock);
139
140	if (wpf->entity.index != 0) {
141		/* Only WPF0 supports flipping. */
142		num_flip_ctrls = 0;
143	} else if (vsp1_feature(vsp1, VSP1_HAS_WPF_HFLIP)) {
144		/*
145		 * When horizontal flip is supported the WPF implements three
146		 * controls (horizontal flip, vertical flip and rotation).
147		 */
148		num_flip_ctrls = 3;
149	} else if (vsp1_feature(vsp1, VSP1_HAS_WPF_VFLIP)) {
150		/*
151		 * When only vertical flip is supported the WPF implements a
152		 * single control (vertical flip).
153		 */
154		num_flip_ctrls = 1;
155	} else {
156		/* Otherwise flipping is not supported. */
157		num_flip_ctrls = 0;
158	}
159
160	ret = vsp1_rwpf_init_ctrls(wpf, num_flip_ctrls);
161	if (ret < 0)
162		return ret;
163
164	if (num_flip_ctrls >= 1) {
165		wpf->flip.ctrls.vflip =
166			v4l2_ctrl_new_std(&wpf->ctrls, &vsp1_wpf_ctrl_ops,
167					  V4L2_CID_VFLIP, 0, 1, 1, 0);
168	}
169
170	if (num_flip_ctrls == 3) {
171		wpf->flip.ctrls.hflip =
172			v4l2_ctrl_new_std(&wpf->ctrls, &vsp1_wpf_ctrl_ops,
173					  V4L2_CID_HFLIP, 0, 1, 1, 0);
174		wpf->flip.ctrls.rotate =
175			v4l2_ctrl_new_std(&wpf->ctrls, &vsp1_wpf_ctrl_ops,
176					  V4L2_CID_ROTATE, 0, 270, 90, 0);
177		v4l2_ctrl_cluster(3, &wpf->flip.ctrls.vflip);
178	}
179
180	if (wpf->ctrls.error)
181		return wpf->ctrls.error;
182
183	return 0;
184}
185
186/* -----------------------------------------------------------------------------
187 * VSP1 Entity Operations
188 */
189
190void vsp1_wpf_stop(struct vsp1_rwpf *wpf)
191{
192	struct vsp1_device *vsp1 = wpf->entity.vsp1;
193
194	/*
195	 * Write to registers directly when stopping the stream as there will be
196	 * no pipeline run to apply the display list.
197	 */
198	vsp1_write(vsp1, VI6_WPF_IRQ_ENB(wpf->entity.index), 0);
199	vsp1_write(vsp1, wpf->entity.index * VI6_WPF_OFFSET +
200		   VI6_WPF_SRCRPF, 0);
201}
202
203static void vsp1_wpf_destroy(struct vsp1_entity *entity)
204{
205	struct vsp1_rwpf *wpf = entity_to_rwpf(entity);
206
207	vsp1_dlm_destroy(wpf->dlm);
208}
209
210static int wpf_configure_writeback_chain(struct vsp1_rwpf *wpf,
211					 struct vsp1_dl_list *dl)
212{
213	unsigned int index = wpf->entity.index;
214	struct vsp1_dl_list *dl_next;
215	struct vsp1_dl_body *dlb;
216
217	dl_next = vsp1_dl_list_get(wpf->dlm);
218	if (!dl_next) {
219		dev_err(wpf->entity.vsp1->dev,
220			"Failed to obtain a dl list, disabling writeback\n");
221		return -ENOMEM;
222	}
223
224	dlb = vsp1_dl_list_get_body0(dl_next);
225	vsp1_dl_body_write(dlb, VI6_WPF_WRBCK_CTRL(index), 0);
226	vsp1_dl_list_add_chain(dl, dl_next);
227
228	return 0;
229}
230
231static void wpf_configure_stream(struct vsp1_entity *entity,
232				 struct v4l2_subdev_state *state,
233				 struct vsp1_pipeline *pipe,
234				 struct vsp1_dl_list *dl,
235				 struct vsp1_dl_body *dlb)
236{
237	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
238	struct vsp1_device *vsp1 = wpf->entity.vsp1;
239	const struct v4l2_mbus_framefmt *source_format;
240	const struct v4l2_mbus_framefmt *sink_format;
241	unsigned int index = wpf->entity.index;
242	unsigned int i;
243	u32 outfmt = 0;
244	u32 srcrpf = 0;
245	int ret;
246
247	sink_format = v4l2_subdev_state_get_format(state, RWPF_PAD_SINK);
248	source_format = v4l2_subdev_state_get_format(state, RWPF_PAD_SOURCE);
249
250	/*
251	 * Format configuration. Skip for IIF (VSPX) or if the pipe doesn't
252	 * write to memory.
253	 */
254	if (!pipe->iif && (!pipe->lif || wpf->writeback)) {
255		const struct v4l2_pix_format_mplane *format = &wpf->format;
256		const struct vsp1_format_info *fmtinfo = wpf->fmtinfo;
257
258		outfmt = fmtinfo->hwfmt << VI6_WPF_OUTFMT_WRFMT_SHIFT;
259
260		if (wpf->flip.rotate)
261			outfmt |= VI6_WPF_OUTFMT_ROT;
262
263		if (fmtinfo->alpha)
264			outfmt |= VI6_WPF_OUTFMT_PXA;
265		if (fmtinfo->swap_yc)
266			outfmt |= VI6_WPF_OUTFMT_SPYCS;
267		if (fmtinfo->swap_uv)
268			outfmt |= VI6_WPF_OUTFMT_SPUVS;
269
270		/* Destination stride and byte swapping. */
271		vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_STRIDE_Y,
272			       format->plane_fmt[0].bytesperline);
273		if (format->num_planes > 1)
274			vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_STRIDE_C,
275				       format->plane_fmt[1].bytesperline);
276
277		vsp1_wpf_write(wpf, dlb, VI6_WPF_DSWAP, fmtinfo->swap);
278
279		if (vsp1_feature(vsp1, VSP1_HAS_WPF_HFLIP) && index == 0)
280			vsp1_wpf_write(wpf, dlb, VI6_WPF_ROT_CTRL,
281				       VI6_WPF_ROT_CTRL_LN16 |
282				       (256 << VI6_WPF_ROT_CTRL_LMEM_WD_SHIFT));
283	}
284
285	if (sink_format->code != source_format->code) {
286		u16 ycbcr_enc;
287		u16 quantization;
288		u32 wrtm;
289
290		if (sink_format->code == MEDIA_BUS_FMT_AYUV8_1X32) {
291			ycbcr_enc = sink_format->ycbcr_enc;
292			quantization = sink_format->quantization;
293		} else {
294			ycbcr_enc = source_format->ycbcr_enc;
295			quantization = source_format->quantization;
296		}
297
298		if (ycbcr_enc == V4L2_YCBCR_ENC_601 &&
299		    quantization == V4L2_QUANTIZATION_LIM_RANGE)
300			wrtm = VI6_WPF_OUTFMT_WRTM_BT601;
301		else if (ycbcr_enc == V4L2_YCBCR_ENC_601 &&
302			 quantization == V4L2_QUANTIZATION_FULL_RANGE)
303			wrtm = VI6_WPF_OUTFMT_WRTM_BT601_EXT;
304		else if (ycbcr_enc == V4L2_YCBCR_ENC_709 &&
305			 quantization == V4L2_QUANTIZATION_LIM_RANGE)
306			wrtm = VI6_WPF_OUTFMT_WRTM_BT709;
307		else
308			wrtm = VI6_WPF_OUTFMT_WRTM_BT709_EXT;
309
310		outfmt |= VI6_WPF_OUTFMT_CSC | wrtm;
311	}
312
313	wpf->outfmt = outfmt;
314
315	vsp1_dl_body_write(dlb, VI6_DPR_WPF_FPORCH(index),
316			   VI6_DPR_WPF_FPORCH_FP_WPFN);
317
318	/*
319	 * Sources. If the pipeline has a single input and BRx is not used,
320	 * configure it as the master layer. Otherwise configure all
321	 * inputs as sub-layers and select the virtual RPF as the master
322	 * layer. For VSPX configure the enabled sources as masters.
323	 */
324	for (i = 0; i < vsp1->info->rpf_count; ++i) {
325		struct vsp1_rwpf *input = pipe->inputs[i];
326
327		if (!input)
328			continue;
329
330		srcrpf |= (pipe->iif || (!pipe->brx && pipe->num_inputs == 1))
331			? VI6_WPF_SRCRPF_RPF_ACT_MST(input->entity.index)
332			: VI6_WPF_SRCRPF_RPF_ACT_SUB(input->entity.index);
333	}
334
335	if (pipe->brx)
336		srcrpf |= pipe->brx->type == VSP1_ENTITY_BRU
337			? VI6_WPF_SRCRPF_VIRACT_MST
338			: VI6_WPF_SRCRPF_VIRACT2_MST;
339
340	vsp1_wpf_write(wpf, dlb, VI6_WPF_SRCRPF, srcrpf);
341
342	/* Enable interrupts. */
343	vsp1_dl_body_write(dlb, VI6_WPF_IRQ_STA(index), 0);
344	vsp1_dl_body_write(dlb, VI6_WPF_IRQ_ENB(index),
345			   VI6_WPF_IRQ_ENB_DFEE);
346
347	if (pipe->iif)
348		return;
349
350	/*
351	 * Configure writeback for display pipelines (the wpf writeback flag is
352	 * never set for memory-to-memory pipelines). Start by adding a chained
353	 * display list to disable writeback after a single frame, and process
354	 * to enable writeback. If the display list allocation fails don't
355	 * enable writeback as we wouldn't be able to safely disable it,
356	 * resulting in possible memory corruption.
357	 */
358	if (wpf->writeback) {
359		ret = wpf_configure_writeback_chain(wpf, dl);
360		if (ret < 0)
361			wpf->writeback = false;
362	}
363
364	vsp1_dl_body_write(dlb, VI6_WPF_WRBCK_CTRL(index),
365			   wpf->writeback ? VI6_WPF_WRBCK_CTRL_WBMD : 0);
366}
367
368static void wpf_configure_frame(struct vsp1_entity *entity,
369				struct vsp1_pipeline *pipe,
370				struct vsp1_dl_list *dl,
371				struct vsp1_dl_body *dlb)
372{
373	const unsigned int mask = BIT(WPF_CTRL_VFLIP)
374				| BIT(WPF_CTRL_HFLIP);
375	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
376	unsigned long flags;
377	u32 outfmt;
378
379	spin_lock_irqsave(&wpf->flip.lock, flags);
380	wpf->flip.active = (wpf->flip.active & ~mask)
381			 | (wpf->flip.pending & mask);
382	spin_unlock_irqrestore(&wpf->flip.lock, flags);
383
384	outfmt = (wpf->alpha << VI6_WPF_OUTFMT_PDV_SHIFT) | wpf->outfmt;
385
386	if (wpf->flip.active & BIT(WPF_CTRL_VFLIP))
387		outfmt |= VI6_WPF_OUTFMT_FLP;
388	if (wpf->flip.active & BIT(WPF_CTRL_HFLIP))
389		outfmt |= VI6_WPF_OUTFMT_HFLP;
390
391	vsp1_wpf_write(wpf, dlb, VI6_WPF_OUTFMT, outfmt);
392}
393
394static void wpf_configure_partition(struct vsp1_entity *entity,
395				    struct vsp1_pipeline *pipe,
396				    const struct vsp1_partition *partition,
397				    struct vsp1_dl_list *dl,
398				    struct vsp1_dl_body *dlb)
399{
400	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
401	struct vsp1_device *vsp1 = wpf->entity.vsp1;
402	struct vsp1_rwpf_memory mem = wpf->mem;
403	const struct v4l2_pix_format_mplane *format = &wpf->format;
404	const struct vsp1_format_info *fmtinfo = wpf->fmtinfo;
405	unsigned int width;
406	unsigned int height;
407	unsigned int left;
408	unsigned int offset;
409	unsigned int flip;
410	unsigned int i;
411
412	/*
413	 * Cropping. The partition algorithm can split the image into multiple
414	 * slices.
415	 */
416	width = partition->wpf.width;
417	left = partition->wpf.left;
418	height = partition->wpf.height;
419
420	vsp1_wpf_write(wpf, dlb, VI6_WPF_HSZCLIP, VI6_WPF_SZCLIP_EN |
421		       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
422		       (width << VI6_WPF_SZCLIP_SIZE_SHIFT));
423	vsp1_wpf_write(wpf, dlb, VI6_WPF_VSZCLIP, VI6_WPF_SZCLIP_EN |
424		       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
425		       (height << VI6_WPF_SZCLIP_SIZE_SHIFT));
426
427	/*
428	 * For display pipelines without writeback enabled there's no memory
429	 * address to configure, return now.
430	 */
431	if (pipe->lif && !wpf->writeback)
432		return;
433
434	/*
435	 * Update the memory offsets based on flipping configuration.
436	 * The destination addresses point to the locations where the
437	 * VSP starts writing to memory, which can be any corner of the
438	 * image depending on the combination of flipping and rotation.
439	 */
440
441	/*
442	 * First take the partition left coordinate into account.
443	 * Compute the offset to order the partitions correctly on the
444	 * output based on whether flipping is enabled. Consider
445	 * horizontal flipping when rotation is disabled but vertical
446	 * flipping when rotation is enabled, as rotating the image
447	 * switches the horizontal and vertical directions. The offset
448	 * is applied horizontally or vertically accordingly.
449	 */
450	flip = wpf->flip.active;
451
452	if (flip & BIT(WPF_CTRL_HFLIP) && !wpf->flip.rotate)
453		offset = format->width - left - width;
454	else if (flip & BIT(WPF_CTRL_VFLIP) && wpf->flip.rotate)
455		offset = format->height - left - width;
456	else
457		offset = left;
458
459	for (i = 0; i < format->num_planes; ++i) {
460		unsigned int hsub = i > 0 ? fmtinfo->hsub : 1;
461		unsigned int vsub = i > 0 ? fmtinfo->vsub : 1;
462
463		if (wpf->flip.rotate)
464			mem.addr[i] += offset / vsub
465				     * format->plane_fmt[i].bytesperline;
466		else
467			mem.addr[i] += offset / hsub
468				     * fmtinfo->bpp[i] / 8;
469	}
470
471	if (flip & BIT(WPF_CTRL_VFLIP)) {
472		/*
473		 * When rotating the output (after rotation) image
474		 * height is equal to the partition width (before
475		 * rotation). Otherwise it is equal to the output
476		 * image height.
477		 */
478		if (wpf->flip.rotate)
479			height = width;
480		else
481			height = format->height;
482
483		mem.addr[0] += (height - 1)
484			     * format->plane_fmt[0].bytesperline;
485
486		if (format->num_planes > 1) {
487			offset = (height / fmtinfo->vsub - 1)
488			       * format->plane_fmt[1].bytesperline;
489			mem.addr[1] += offset;
490			mem.addr[2] += offset;
491		}
492	}
493
494	if (wpf->flip.rotate && !(flip & BIT(WPF_CTRL_HFLIP))) {
495		unsigned int hoffset = max(0, (int)format->width - 16);
496
497		/*
498		 * Compute the output coordinate. The partition
499		 * horizontal (left) offset becomes a vertical offset.
500		 */
501		for (i = 0; i < format->num_planes; ++i) {
502			unsigned int hsub = i > 0 ? fmtinfo->hsub : 1;
503
504			mem.addr[i] += hoffset / hsub
505				     * fmtinfo->bpp[i] / 8;
506		}
507	}
508
509	/*
510	 * On Gen3+ hardware the SPUVS bit has no effect on 3-planar
511	 * formats. Swap the U and V planes manually in that case.
512	 */
513	if (vsp1->info->gen >= 3 && format->num_planes == 3 &&
514	    fmtinfo->swap_uv)
515		swap(mem.addr[1], mem.addr[2]);
516
517	vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_ADDR_Y, mem.addr[0]);
518	vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_ADDR_C0, mem.addr[1]);
519	vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_ADDR_C1, mem.addr[2]);
520
521	/*
522	 * Writeback operates in single-shot mode and lasts for a single frame,
523	 * reset the writeback flag to false for the next frame.
524	 */
525	wpf->writeback = false;
526}
527
528static unsigned int wpf_max_width(struct vsp1_entity *entity,
529				  struct v4l2_subdev_state *state,
530				  struct vsp1_pipeline *pipe)
531{
532	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
533
534	return wpf->flip.rotate ? 256 : wpf->max_width;
535}
536
537static void wpf_partition(struct vsp1_entity *entity,
538			  struct v4l2_subdev_state *state,
539			  struct vsp1_pipeline *pipe,
540			  struct vsp1_partition *partition,
541			  unsigned int partition_idx,
542			  struct v4l2_rect *window)
543{
544	partition->wpf = *window;
545}
546
547static const struct vsp1_entity_operations wpf_entity_ops = {
548	.destroy = vsp1_wpf_destroy,
549	.configure_stream = wpf_configure_stream,
550	.configure_frame = wpf_configure_frame,
551	.configure_partition = wpf_configure_partition,
552	.max_width = wpf_max_width,
553	.partition = wpf_partition,
554};
555
556/* -----------------------------------------------------------------------------
557 * Initialization and Cleanup
558 */
559
560struct vsp1_rwpf *vsp1_wpf_create(struct vsp1_device *vsp1, unsigned int index)
561{
562	struct vsp1_rwpf *wpf;
563	char name[6];
564	int ret;
565
566	wpf = devm_kzalloc(vsp1->dev, sizeof(*wpf), GFP_KERNEL);
567	if (wpf == NULL)
568		return ERR_PTR(-ENOMEM);
569
570	if (vsp1->info->gen == 2) {
571		wpf->max_width = WPF_GEN2_MAX_WIDTH;
572		wpf->max_height = WPF_GEN2_MAX_HEIGHT;
573	} else {
574		wpf->max_width = WPF_GEN3_MAX_WIDTH;
575		wpf->max_height = WPF_GEN3_MAX_HEIGHT;
576	}
577
578	wpf->entity.ops = &wpf_entity_ops;
579	wpf->entity.type = VSP1_ENTITY_WPF;
580	wpf->entity.index = index;
581
582	sprintf(name, "wpf.%u", index);
583	ret = vsp1_entity_init(vsp1, &wpf->entity, name, 2, &vsp1_rwpf_subdev_ops,
584			       MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER);
585	if (ret < 0)
586		return ERR_PTR(ret);
587
588	/* Initialize the display list manager. */
589	wpf->dlm = vsp1_dlm_create(vsp1, index, 64);
590	if (!wpf->dlm) {
591		ret = -ENOMEM;
592		goto error;
593	}
594
595	/* Initialize the control handler. */
596	ret = wpf_init_controls(wpf);
597	if (ret < 0) {
598		dev_err(vsp1->dev, "wpf%u: failed to initialize controls\n",
599			index);
600		goto error;
601	}
602
603	v4l2_ctrl_handler_setup(&wpf->ctrls);
604
605	return wpf;
606
607error:
608	vsp1_entity_destroy(&wpf->entity);
609	return ERR_PTR(ret);
610}