mali-c55-core.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * ARM Mali-C55 ISP Driver - Core driver code
  4 *
  5 * Copyright (C) 2025 Ideas on Board Oy
  6 */
  7
  8#include <linux/bitops.h>
  9#include <linux/cleanup.h>
 10#include <linux/clk.h>
 11#include <linux/delay.h>
 12#include <linux/device.h>
 13#include <linux/interrupt.h>
 14#include <linux/iopoll.h>
 15#include <linux/ioport.h>
 16#include <linux/mod_devicetable.h>
 17#include <linux/of.h>
 18#include <linux/of_reserved_mem.h>
 19#include <linux/platform_device.h>
 20#include <linux/pm_runtime.h>
 21#include <linux/reset.h>
 22#include <linux/slab.h>
 23#include <linux/string.h>
 24
 25#include <media/media-entity.h>
 26#include <media/v4l2-device.h>
 27#include <media/v4l2-mc.h>
 28#include <media/videobuf2-core.h>
 29#include <media/videobuf2-dma-contig.h>
 30
 31#include "mali-c55-common.h"
 32#include "mali-c55-registers.h"
 33
 34static const char * const mali_c55_interrupt_names[] = {
 35	[MALI_C55_IRQ_ISP_START] = "ISP start",
 36	[MALI_C55_IRQ_ISP_DONE] = "ISP done",
 37	[MALI_C55_IRQ_MCM_ERROR] = "Multi-context management error",
 38	[MALI_C55_IRQ_BROKEN_FRAME_ERROR] = "Broken frame error",
 39	[MALI_C55_IRQ_MET_AF_DONE] = "AF metering done",
 40	[MALI_C55_IRQ_MET_AEXP_DONE] = "AEXP metering done",
 41	[MALI_C55_IRQ_MET_AWB_DONE] = "AWB metering done",
 42	[MALI_C55_IRQ_AEXP_1024_DONE] = "AEXP 1024-bit histogram done",
 43	[MALI_C55_IRQ_IRIDIX_MET_DONE] = "Iridix metering done",
 44	[MALI_C55_IRQ_LUT_INIT_DONE] = "LUT memory init done",
 45	[MALI_C55_IRQ_FR_Y_DONE] = "Full resolution Y plane DMA done",
 46	[MALI_C55_IRQ_FR_UV_DONE] = "Full resolution U/V plane DMA done",
 47	[MALI_C55_IRQ_DS_Y_DONE] = "Downscale Y plane DMA done",
 48	[MALI_C55_IRQ_DS_UV_DONE] = "Downscale U/V plane DMA done",
 49	[MALI_C55_IRQ_LINEARIZATION_DONE] = "Linearisation done",
 50	[MALI_C55_IRQ_RAW_FRONTEND_DONE] = "Raw frontend processing done",
 51	[MALI_C55_IRQ_NOISE_REDUCTION_DONE] = "Noise reduction done",
 52	[MALI_C55_IRQ_IRIDIX_DONE] = "Iridix done",
 53	[MALI_C55_IRQ_BAYER2RGB_DONE] = "Bayer to RGB conversion done",
 54	[MALI_C55_IRQ_WATCHDOG_TIMER] = "Watchdog timer timed out",
 55	[MALI_C55_IRQ_FRAME_COLLISION] = "Frame collision error",
 56	[MALI_C55_IRQ_UNUSED] = "IRQ bit unused",
 57	[MALI_C55_IRQ_DMA_ERROR] = "DMA error",
 58	[MALI_C55_IRQ_INPUT_STOPPED] = "Input port safely stopped",
 59	[MALI_C55_IRQ_MET_AWB_TARGET1_HIT] = "AWB metering target 1 address hit",
 60	[MALI_C55_IRQ_MET_AWB_TARGET2_HIT] = "AWB metering target 2 address hit"
 61};
 62
 63static const unsigned int config_space_addrs[] = {
 64	[MALI_C55_CONFIG_PING] = 0x0ab6c,
 65	[MALI_C55_CONFIG_PONG] = 0x22b2c,
 66};
 67
 68static const char * const mali_c55_clk_names[MALI_C55_NUM_CLKS] = {
 69	"vclk",
 70	"aclk",
 71	"hclk",
 72};
 73
 74static const char * const mali_c55_reset_names[MALI_C55_NUM_RESETS] = {
 75	"vresetn",
 76	"aresetn",
 77	"hresetn",
 78};
 79
 80/*
 81 * System IO
 82 *
 83 * The Mali-C55 ISP has up to two configuration register spaces (called 'ping'
 84 * and 'pong'), with the expectation that the 'active' space will be left
 85 * untouched whilst a frame is being processed and the 'inactive' space
 86 * configured ready to be switched to during the blanking period before the next
 87 * frame processing starts. These spaces should ideally be set via DMA transfer
 88 * from a buffer rather than through individual register set operations. There
 89 * is also a shared global register space which should be set normally. For now
 90 * though we will simply use a CPU write and target DMA transfers of the config
 91 * space in the future.
 92 *
 93 * As groundwork for that path any read/write call that is made to an address
 94 * within those config spaces should infact be directed to a buffer that was
 95 * allocated to hold them rather than the IO memory itself. The actual copy of
 96 * that buffer to IO mem will happen on interrupt.
 97 */
 98
 99void mali_c55_write(struct mali_c55 *mali_c55, unsigned int addr, u32 val)
100{
101	WARN_ON(addr >= MALI_C55_REG_CONFIG_SPACES_OFFSET);
102
103	writel(val, mali_c55->base + addr);
104}
105
106u32 mali_c55_read(struct mali_c55 *mali_c55, unsigned int addr)
107{
108	WARN_ON(addr >= MALI_C55_REG_CONFIG_SPACES_OFFSET);
109
110	return readl(mali_c55->base + addr);
111}
112
113void mali_c55_update_bits(struct mali_c55 *mali_c55, unsigned int addr,
114			  u32 mask, u32 val)
115{
116	u32 orig, new;
117
118	orig = mali_c55_read(mali_c55, addr);
119
120	new = orig & ~mask;
121	new |= val & mask;
122
123	if (new != orig)
124		mali_c55_write(mali_c55, addr, new);
125}
126
127static void __mali_c55_ctx_write(struct mali_c55_context *ctx,
128				 unsigned int addr, u32 val)
129{
130	addr = (addr - MALI_C55_REG_CONFIG_SPACES_OFFSET) / 4;
131	ctx->registers[addr] = val;
132}
133
134void mali_c55_ctx_write(struct mali_c55 *mali_c55, unsigned int addr, u32 val)
135{
136	struct mali_c55_context *ctx = mali_c55_get_active_context(mali_c55);
137
138	WARN_ON(addr < MALI_C55_REG_CONFIG_SPACES_OFFSET);
139
140	spin_lock(&ctx->lock);
141	__mali_c55_ctx_write(ctx, addr, val);
142	spin_unlock(&ctx->lock);
143}
144
145static u32 __mali_c55_ctx_read(struct mali_c55_context *ctx, unsigned int addr)
146{
147	addr = (addr - MALI_C55_REG_CONFIG_SPACES_OFFSET) / 4;
148	return ctx->registers[addr];
149}
150
151u32 mali_c55_ctx_read(struct mali_c55 *mali_c55, unsigned int addr)
152{
153	struct mali_c55_context *ctx = mali_c55_get_active_context(mali_c55);
154	u32 val;
155
156	WARN_ON(addr < MALI_C55_REG_CONFIG_SPACES_OFFSET);
157
158	spin_lock(&ctx->lock);
159	val = __mali_c55_ctx_read(ctx, addr);
160	spin_unlock(&ctx->lock);
161
162	return val;
163}
164
165void mali_c55_ctx_update_bits(struct mali_c55 *mali_c55, unsigned int addr,
166			      u32 mask, u32 val)
167{
168	struct mali_c55_context *ctx = mali_c55_get_active_context(mali_c55);
169	u32 orig, tmp;
170
171	WARN_ON(addr < MALI_C55_REG_CONFIG_SPACES_OFFSET);
172
173	spin_lock(&ctx->lock);
174
175	orig = __mali_c55_ctx_read(ctx, addr);
176
177	tmp = orig & ~mask;
178	tmp |= val & mask;
179
180	if (tmp != orig)
181		__mali_c55_ctx_write(ctx, addr, tmp);
182
183	spin_unlock(&ctx->lock);
184}
185
186int mali_c55_config_write(struct mali_c55_context *ctx,
187			  enum mali_c55_config_spaces cfg_space,
188			  bool force_synchronous)
189{
190	struct mali_c55 *mali_c55 = ctx->mali_c55;
191
192	memcpy_toio(mali_c55->base + config_space_addrs[cfg_space],
193		    ctx->registers, MALI_C55_CONFIG_SPACE_SIZE);
194
195	return 0;
196}
197
198struct mali_c55_context *mali_c55_get_active_context(struct mali_c55 *mali_c55)
199{
200	return &mali_c55->context;
201}
202
203static void mali_c55_remove_links(struct mali_c55 *mali_c55)
204{
205	unsigned int i;
206
207	media_entity_remove_links(&mali_c55->tpg.sd.entity);
208	media_entity_remove_links(&mali_c55->isp.sd.entity);
209
210	for (i = 0; i < MALI_C55_NUM_RSZS; i++)
211		media_entity_remove_links(&mali_c55->resizers[i].sd.entity);
212
213	for (i = 0; i < MALI_C55_NUM_CAP_DEVS; i++)
214		media_entity_remove_links(&mali_c55->cap_devs[i].vdev.entity);
215}
216
217static int mali_c55_create_links(struct mali_c55 *mali_c55)
218{
219	int ret;
220
221	/* Test pattern generator to ISP */
222	ret = media_create_pad_link(&mali_c55->tpg.sd.entity, 0,
223				    &mali_c55->isp.sd.entity,
224				    MALI_C55_ISP_PAD_SINK_VIDEO, 0);
225	if (ret) {
226		dev_err(mali_c55->dev, "failed to link TPG and ISP\n");
227		goto err_remove_links;
228	}
229
230	/* Full resolution resizer pipe. */
231	ret = media_create_pad_link(&mali_c55->isp.sd.entity,
232			MALI_C55_ISP_PAD_SOURCE_VIDEO,
233			&mali_c55->resizers[MALI_C55_RSZ_FR].sd.entity,
234			MALI_C55_RSZ_SINK_PAD,
235			MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE);
236	if (ret) {
237		dev_err(mali_c55->dev, "failed to link ISP and FR resizer\n");
238		goto err_remove_links;
239	}
240
241	/* Full resolution bypass. */
242	ret = media_create_pad_link(&mali_c55->isp.sd.entity,
243				    MALI_C55_ISP_PAD_SOURCE_BYPASS,
244				    &mali_c55->resizers[MALI_C55_RSZ_FR].sd.entity,
245				    MALI_C55_RSZ_SINK_BYPASS_PAD,
246				    MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE);
247	if (ret) {
248		dev_err(mali_c55->dev, "failed to link ISP and FR resizer\n");
249		goto err_remove_links;
250	}
251
252	/* Resizer pipe to video capture nodes. */
253	ret = media_create_pad_link(&mali_c55->resizers[0].sd.entity,
254			MALI_C55_RSZ_SOURCE_PAD,
255			&mali_c55->cap_devs[MALI_C55_CAP_DEV_FR].vdev.entity,
256			0, MEDIA_LNK_FL_ENABLED);
257	if (ret) {
258		dev_err(mali_c55->dev,
259			"failed to link FR resizer and video device\n");
260		goto err_remove_links;
261	}
262
263	/* The downscale pipe is an optional hardware block */
264	if (mali_c55->capabilities & MALI_C55_GPS_DS_PIPE_FITTED) {
265		ret = media_create_pad_link(&mali_c55->isp.sd.entity,
266			MALI_C55_ISP_PAD_SOURCE_VIDEO,
267			&mali_c55->resizers[MALI_C55_RSZ_DS].sd.entity,
268			MALI_C55_RSZ_SINK_PAD,
269			MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE);
270		if (ret) {
271			dev_err(mali_c55->dev,
272				"failed to link ISP and DS resizer\n");
273			goto err_remove_links;
274		}
275
276		ret = media_create_pad_link(&mali_c55->resizers[1].sd.entity,
277			MALI_C55_RSZ_SOURCE_PAD,
278			&mali_c55->cap_devs[MALI_C55_CAP_DEV_DS].vdev.entity,
279			0, MEDIA_LNK_FL_ENABLED);
280		if (ret) {
281			dev_err(mali_c55->dev,
282				"failed to link DS resizer and video device\n");
283			goto err_remove_links;
284		}
285	}
286
287	ret = media_create_pad_link(&mali_c55->isp.sd.entity,
288				    MALI_C55_ISP_PAD_SOURCE_STATS,
289				    &mali_c55->stats.vdev.entity, 0,
290				    MEDIA_LNK_FL_ENABLED);
291	if (ret) {
292		dev_err(mali_c55->dev,
293			"failed to link ISP and 3a stats node\n");
294		goto err_remove_links;
295	}
296
297	ret = media_create_pad_link(&mali_c55->params.vdev.entity, 0,
298				    &mali_c55->isp.sd.entity,
299				    MALI_C55_ISP_PAD_SINK_PARAMS,
300				    MEDIA_LNK_FL_ENABLED);
301	if (ret) {
302		dev_err(mali_c55->dev,
303			"failed to link ISP and parameters video node\n");
304		goto err_remove_links;
305	}
306
307	return 0;
308
309err_remove_links:
310	mali_c55_remove_links(mali_c55);
311	return ret;
312}
313
314static void mali_c55_unregister_entities(struct mali_c55 *mali_c55)
315{
316	mali_c55_remove_links(mali_c55);
317	mali_c55_unregister_tpg(mali_c55);
318	mali_c55_unregister_isp(mali_c55);
319	mali_c55_unregister_resizers(mali_c55);
320	mali_c55_unregister_capture_devs(mali_c55);
321	mali_c55_unregister_params(mali_c55);
322	mali_c55_unregister_stats(mali_c55);
323}
324
325static void mali_c55_swap_next_config(struct mali_c55 *mali_c55)
326{
327	struct mali_c55_context *ctx = mali_c55_get_active_context(mali_c55);
328
329	mali_c55_config_write(ctx, mali_c55->next_config ?
330			      MALI_C55_CONFIG_PING : MALI_C55_CONFIG_PONG,
331			      false);
332
333	mali_c55_update_bits(mali_c55, MALI_C55_REG_MCU_CONFIG,
334		MALI_C55_REG_MCU_CONFIG_WRITE_MASK,
335		MALI_C55_MCU_CONFIG_WRITE(mali_c55->next_config));
336}
337
338static int mali_c55_register_entities(struct mali_c55 *mali_c55)
339{
340	int ret;
341
342	ret = mali_c55_register_tpg(mali_c55);
343	if (ret)
344		return ret;
345
346	ret = mali_c55_register_isp(mali_c55);
347	if (ret)
348		goto err_unregister_entities;
349
350	ret = mali_c55_register_resizers(mali_c55);
351	if (ret)
352		goto err_unregister_entities;
353
354	ret = mali_c55_register_capture_devs(mali_c55);
355	if (ret)
356		goto err_unregister_entities;
357
358	ret = mali_c55_register_params(mali_c55);
359	if (ret)
360		goto err_unregister_entities;
361
362	ret = mali_c55_register_stats(mali_c55);
363	if (ret)
364		goto err_unregister_entities;
365
366	ret = mali_c55_create_links(mali_c55);
367	if (ret)
368		goto err_unregister_entities;
369
370	return 0;
371
372err_unregister_entities:
373	mali_c55_unregister_entities(mali_c55);
374
375	return ret;
376}
377
378static int mali_c55_notifier_bound(struct v4l2_async_notifier *notifier,
379				   struct v4l2_subdev *subdev,
380				   struct v4l2_async_connection *asc)
381{
382	struct mali_c55 *mali_c55 = container_of(notifier,
383						struct mali_c55, notifier);
384	struct media_pad *pad = &mali_c55->isp.pads[MALI_C55_ISP_PAD_SINK_VIDEO];
385	int ret;
386
387	/*
388	 * By default we'll flag this link enabled and the TPG disabled, but
389	 * no immutable flag because we need to be able to switch between the
390	 * two.
391	 */
392	ret = v4l2_create_fwnode_links_to_pad(subdev, pad,
393					      MEDIA_LNK_FL_ENABLED);
394	if (ret)
395		dev_err(mali_c55->dev, "failed to create link for %s\n",
396			subdev->name);
397
398	return ret;
399}
400
401static int mali_c55_notifier_complete(struct v4l2_async_notifier *notifier)
402{
403	struct mali_c55 *mali_c55 = container_of(notifier,
404						struct mali_c55, notifier);
405
406	return v4l2_device_register_subdev_nodes(&mali_c55->v4l2_dev);
407}
408
409static const struct v4l2_async_notifier_operations mali_c55_notifier_ops = {
410	.bound = mali_c55_notifier_bound,
411	.complete = mali_c55_notifier_complete,
412};
413
414static int mali_c55_parse_endpoint(struct mali_c55 *mali_c55)
415{
416	struct v4l2_async_connection *asc;
417	struct fwnode_handle *ep;
418
419	/*
420	 * The ISP should have a single endpoint pointing to some flavour of
421	 * CSI-2 receiver...but for now at least we do want everything to work
422	 * normally even with no sensors connected, as we have the TPG. If we
423	 * don't find a sensor just warn and return success.
424	 */
425	ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(mali_c55->dev),
426					     0, 0, 0);
427	if (!ep) {
428		dev_warn(mali_c55->dev, "no local endpoint found\n");
429		return 0;
430	}
431
432	asc = v4l2_async_nf_add_fwnode_remote(&mali_c55->notifier, ep,
433					      struct v4l2_async_connection);
434	fwnode_handle_put(ep);
435	if (IS_ERR(asc)) {
436		dev_err(mali_c55->dev, "failed to add remote fwnode\n");
437		return PTR_ERR(asc);
438	}
439
440	return 0;
441}
442
443static int mali_c55_media_frameworks_init(struct mali_c55 *mali_c55)
444{
445	int ret;
446
447	strscpy(mali_c55->media_dev.model, "ARM Mali-C55 ISP",
448		sizeof(mali_c55->media_dev.model));
449
450	media_device_init(&mali_c55->media_dev);
451
452	ret = media_device_register(&mali_c55->media_dev);
453	if (ret)
454		goto err_cleanup_media_device;
455
456	mali_c55->v4l2_dev.mdev = &mali_c55->media_dev;
457	ret = v4l2_device_register(mali_c55->dev, &mali_c55->v4l2_dev);
458	if (ret) {
459		dev_err(mali_c55->dev, "failed to register V4L2 device\n");
460		goto err_unregister_media_device;
461	};
462
463	mali_c55->notifier.ops = &mali_c55_notifier_ops;
464	v4l2_async_nf_init(&mali_c55->notifier, &mali_c55->v4l2_dev);
465
466	ret = mali_c55_register_entities(mali_c55);
467	if (ret) {
468		dev_err(mali_c55->dev, "failed to register entities\n");
469		goto err_cleanup_nf;
470	}
471
472	ret = mali_c55_parse_endpoint(mali_c55);
473	if (ret)
474		goto err_cleanup_nf;
475
476	ret = v4l2_async_nf_register(&mali_c55->notifier);
477	if (ret) {
478		dev_err(mali_c55->dev, "failed to register notifier\n");
479		goto err_unregister_entities;
480	}
481
482	return 0;
483
484err_unregister_entities:
485	mali_c55_unregister_entities(mali_c55);
486err_cleanup_nf:
487	v4l2_async_nf_cleanup(&mali_c55->notifier);
488	v4l2_device_unregister(&mali_c55->v4l2_dev);
489err_unregister_media_device:
490	media_device_unregister(&mali_c55->media_dev);
491err_cleanup_media_device:
492	media_device_cleanup(&mali_c55->media_dev);
493
494	return ret;
495}
496
497static void mali_c55_media_frameworks_deinit(struct mali_c55 *mali_c55)
498{
499	v4l2_async_nf_unregister(&mali_c55->notifier);
500	mali_c55_unregister_entities(mali_c55);
501	v4l2_async_nf_cleanup(&mali_c55->notifier);
502	v4l2_device_unregister(&mali_c55->v4l2_dev);
503	media_device_unregister(&mali_c55->media_dev);
504	media_device_cleanup(&mali_c55->media_dev);
505}
506
507bool mali_c55_pipeline_ready(struct mali_c55 *mali_c55)
508{
509	struct mali_c55_cap_dev *fr = &mali_c55->cap_devs[MALI_C55_CAP_DEV_FR];
510	struct mali_c55_cap_dev *ds = &mali_c55->cap_devs[MALI_C55_CAP_DEV_DS];
511	struct mali_c55_params *params = &mali_c55->params;
512	struct mali_c55_stats *stats = &mali_c55->stats;
513
514	return vb2_start_streaming_called(&fr->queue) &&
515	       (!(mali_c55->capabilities & MALI_C55_GPS_DS_PIPE_FITTED) ||
516		vb2_start_streaming_called(&ds->queue)) &&
517	       vb2_start_streaming_called(&params->queue) &&
518	       vb2_start_streaming_called(&stats->queue);
519}
520
521static int mali_c55_check_hwcfg(struct mali_c55 *mali_c55)
522{
523	u32 product, version, revision, capabilities;
524
525	product = mali_c55_read(mali_c55, MALI_C55_REG_PRODUCT);
526	version = mali_c55_read(mali_c55, MALI_C55_REG_VERSION);
527	revision = mali_c55_read(mali_c55, MALI_C55_REG_REVISION);
528
529	mali_c55->media_dev.hw_revision = version;
530
531	dev_info(mali_c55->dev, "Detected Mali-C55 ISP %u.%u.%u\n",
532		 product, version, revision);
533
534	capabilities = mali_c55_read(mali_c55,
535				     MALI_C55_REG_GLOBAL_PARAMETER_STATUS);
536
537	/*
538	 * In its current iteration, the driver only supports inline mode. Given
539	 * we cannot control input data timing in this mode, we cannot guarantee
540	 * that the vertical blanking periods between frames will be long enough
541	 * for us to write configuration data to the ISP during them. For that
542	 * reason we can't really support single config space configuration
543	 * until memory input mode is implemented.
544	 */
545	if (!(capabilities & MALI_C55_GPS_PONG_FITTED)) {
546		dev_err(mali_c55->dev, "Pong config space not fitted.\n");
547		return -EINVAL;
548	}
549
550	mali_c55->capabilities = capabilities & 0xffff;
551
552	return 0;
553}
554
555static irqreturn_t mali_c55_isr(int irq, void *context)
556{
557	struct device *dev = context;
558	struct mali_c55 *mali_c55 = dev_get_drvdata(dev);
559	unsigned long interrupt_status;
560	u32 curr_config;
561	unsigned int i;
562
563	interrupt_status = mali_c55_read(mali_c55,
564					 MALI_C55_REG_INTERRUPT_STATUS_VECTOR);
565	if (!interrupt_status)
566		return IRQ_NONE;
567
568	mali_c55_write(mali_c55, MALI_C55_REG_INTERRUPT_CLEAR_VECTOR,
569		       interrupt_status);
570	mali_c55_write(mali_c55, MALI_C55_REG_INTERRUPT_CLEAR, 1);
571	mali_c55_write(mali_c55, MALI_C55_REG_INTERRUPT_CLEAR, 0);
572
573	for_each_set_bit(i, &interrupt_status, MALI_C55_NUM_IRQ_BITS) {
574		switch (i) {
575		case MALI_C55_IRQ_ISP_START:
576			mali_c55_isp_queue_event_sof(mali_c55);
577
578			mali_c55_set_next_buffer(&mali_c55->cap_devs[MALI_C55_CAP_DEV_FR]);
579			if (mali_c55->capabilities & MALI_C55_GPS_DS_PIPE_FITTED)
580				mali_c55_set_next_buffer(&mali_c55->cap_devs[MALI_C55_CAP_DEV_DS]);
581
582			/*
583			 * When the ISP starts a frame we have some work to do:
584			 *
585			 * 1. Copy over the config for the **next** frame
586			 * 2. Read out the metering stats for the **last** frame
587			 */
588
589			curr_config = mali_c55_read(mali_c55,
590						    MALI_C55_REG_PING_PONG_READ);
591			curr_config &= MALI_C55_REG_PING_PONG_READ_MASK;
592			curr_config >>= ffs(MALI_C55_REG_PING_PONG_READ_MASK) - 1;
593			mali_c55->next_config = curr_config ^ 1;
594
595			/*
596			 * Write the configuration parameters received from
597			 * userspace into the configuration buffer, which will
598			 * be transferred to the 'next' active config space at
599			 * by mali_c55_swap_next_config().
600			 */
601			mali_c55_params_write_config(mali_c55);
602
603			mali_c55_stats_fill_buffer(mali_c55,
604						   mali_c55->next_config ^ 1);
605
606			mali_c55_swap_next_config(mali_c55);
607
608			break;
609		case MALI_C55_IRQ_ISP_DONE:
610			/*
611			 * TODO: Where the ISP has no Pong config fitted, we'd
612			 * have to do the mali_c55_swap_next_config() call here.
613			 */
614			break;
615		case MALI_C55_IRQ_FR_Y_DONE:
616			mali_c55_set_plane_done(&mali_c55->cap_devs[MALI_C55_CAP_DEV_FR],
617						MALI_C55_PLANE_Y);
618			break;
619		case MALI_C55_IRQ_FR_UV_DONE:
620			mali_c55_set_plane_done(&mali_c55->cap_devs[MALI_C55_CAP_DEV_FR],
621						MALI_C55_PLANE_UV);
622			break;
623		case MALI_C55_IRQ_DS_Y_DONE:
624			mali_c55_set_plane_done(&mali_c55->cap_devs[MALI_C55_CAP_DEV_DS],
625						MALI_C55_PLANE_Y);
626			break;
627		case MALI_C55_IRQ_DS_UV_DONE:
628			mali_c55_set_plane_done(&mali_c55->cap_devs[MALI_C55_CAP_DEV_DS],
629						MALI_C55_PLANE_UV);
630			break;
631		default:
632			/*
633			 * Only the above interrupts are currently unmasked. If
634			 * we receive anything else here then something weird
635			 * has gone on.
636			 */
637			dev_err(dev, "masked interrupt %s triggered\n",
638				mali_c55_interrupt_names[i]);
639		}
640	}
641
642	return IRQ_HANDLED;
643}
644
645static int mali_c55_init_context(struct mali_c55 *mali_c55,
646				 struct resource *res)
647{
648	struct mali_c55_context *ctx = &mali_c55->context;
649
650	ctx->base = res->start;
651	ctx->mali_c55 = mali_c55;
652	spin_lock_init(&ctx->lock);
653
654	ctx->registers = kzalloc(MALI_C55_CONFIG_SPACE_SIZE, GFP_KERNEL);
655	if (!ctx->registers)
656		return -ENOMEM;
657
658	/*
659	 * The allocated memory is empty, we need to load the default
660	 * register settings. We just read Ping; it's identical to Pong.
661	 */
662	memcpy_fromio(ctx->registers,
663		      mali_c55->base + config_space_addrs[MALI_C55_CONFIG_PING],
664		      MALI_C55_CONFIG_SPACE_SIZE);
665
666	/*
667	 * Some features of the ISP need to be disabled by default and only
668	 * enabled at the same time as they're configured by a parameters buffer
669	 */
670
671	/* Bypass the sqrt and square compression and expansion modules */
672	mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BYPASS_1,
673				 MALI_C55_REG_BYPASS_1_FE_SQRT,
674				 MALI_C55_REG_BYPASS_1_FE_SQRT);
675	mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BYPASS_3,
676				 MALI_C55_REG_BYPASS_3_SQUARE_BE,
677				 MALI_C55_REG_BYPASS_3_SQUARE_BE);
678
679	/* Bypass the temper module */
680	mali_c55_ctx_write(mali_c55, MALI_C55_REG_BYPASS_2,
681			   MALI_C55_REG_BYPASS_2_TEMPER);
682
683	/* Disable the temper module's DMA read/write */
684	mali_c55_ctx_write(mali_c55, MALI_C55_REG_TEMPER_DMA_IO, 0x0);
685
686	/* Bypass the colour noise reduction  */
687	mali_c55_ctx_write(mali_c55, MALI_C55_REG_BYPASS_4,
688			   MALI_C55_REG_BYPASS_4_CNR);
689
690	/* Disable the sinter module */
691	mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_SINTER_CONFIG,
692				 MALI_C55_SINTER_ENABLE_MASK, 0);
693
694	/* Disable the RGB Gamma module for each output */
695	mali_c55_ctx_write(mali_c55, MALI_C55_REG_FR_GAMMA_RGB_ENABLE, 0);
696	mali_c55_ctx_write(mali_c55, MALI_C55_REG_DS_GAMMA_RGB_ENABLE, 0);
697
698	/* Disable the colour correction matrix */
699	mali_c55_ctx_write(mali_c55, MALI_C55_REG_CCM_ENABLE, 0);
700
701	return 0;
702}
703
704static void __mali_c55_power_off(struct mali_c55 *mali_c55)
705{
706	reset_control_bulk_assert(ARRAY_SIZE(mali_c55->resets), mali_c55->resets);
707	clk_bulk_disable_unprepare(ARRAY_SIZE(mali_c55->clks), mali_c55->clks);
708}
709
710static int __maybe_unused mali_c55_runtime_suspend(struct device *dev)
711{
712	struct mali_c55 *mali_c55 = dev_get_drvdata(dev);
713
714	if (irq_has_action(mali_c55->irqnum))
715		free_irq(mali_c55->irqnum, dev);
716	__mali_c55_power_off(mali_c55);
717
718	return 0;
719}
720
721static int __mali_c55_power_on(struct mali_c55 *mali_c55)
722{
723	int ret;
724	u32 val;
725
726	ret = clk_bulk_prepare_enable(ARRAY_SIZE(mali_c55->clks),
727				      mali_c55->clks);
728	if (ret) {
729		dev_err(mali_c55->dev, "failed to enable clocks\n");
730		return ret;
731	}
732
733	ret = reset_control_bulk_deassert(ARRAY_SIZE(mali_c55->resets),
734					  mali_c55->resets);
735	if (ret) {
736		dev_err(mali_c55->dev, "failed to deassert resets\n");
737		return ret;
738	}
739
740	/* Use "software only" context management. */
741	mali_c55_update_bits(mali_c55, MALI_C55_REG_MCU_CONFIG,
742			     MALI_C55_REG_MCU_CONFIG_OVERRIDE_MASK, 0x01);
743
744	/*
745	 * Mask the interrupts and clear any that were set, then unmask the ones
746	 * that we actually want to handle.
747	 */
748	mali_c55_write(mali_c55, MALI_C55_REG_INTERRUPT_MASK_VECTOR,
749		       MALI_C55_INTERRUPT_MASK_ALL);
750	mali_c55_write(mali_c55, MALI_C55_REG_INTERRUPT_CLEAR_VECTOR,
751		       MALI_C55_INTERRUPT_MASK_ALL);
752	mali_c55_write(mali_c55, MALI_C55_REG_INTERRUPT_CLEAR, 0x01);
753	mali_c55_write(mali_c55, MALI_C55_REG_INTERRUPT_CLEAR, 0x00);
754
755	mali_c55_update_bits(mali_c55, MALI_C55_REG_INTERRUPT_MASK_VECTOR,
756			     MALI_C55_INTERRUPT_BIT(MALI_C55_IRQ_ISP_START) |
757			     MALI_C55_INTERRUPT_BIT(MALI_C55_IRQ_ISP_DONE) |
758			     MALI_C55_INTERRUPT_BIT(MALI_C55_IRQ_FR_Y_DONE) |
759			     MALI_C55_INTERRUPT_BIT(MALI_C55_IRQ_FR_UV_DONE) |
760			     MALI_C55_INTERRUPT_BIT(MALI_C55_IRQ_DS_Y_DONE) |
761			     MALI_C55_INTERRUPT_BIT(MALI_C55_IRQ_DS_UV_DONE),
762			     0x00);
763
764	/* Set safe stop to ensure we're in a non-streaming state */
765	mali_c55_write(mali_c55, MALI_C55_REG_INPUT_MODE_REQUEST,
766		       MALI_C55_INPUT_SAFE_STOP);
767	readl_poll_timeout(mali_c55->base + MALI_C55_REG_MODE_STATUS,
768			   val, !val, 10 * USEC_PER_MSEC, 250 * USEC_PER_MSEC);
769
770	return 0;
771}
772
773static int __maybe_unused mali_c55_runtime_resume(struct device *dev)
774{
775	struct mali_c55 *mali_c55 = dev_get_drvdata(dev);
776	int ret;
777
778	ret = __mali_c55_power_on(mali_c55);
779	if (ret)
780		return ret;
781
782	/*
783	 * The driver needs to transfer large amounts of register settings to
784	 * the ISP each frame, using either a DMA transfer or memcpy. We use a
785	 * threaded IRQ to avoid disabling interrupts the entire time that's
786	 * happening.
787	 */
788	ret = request_threaded_irq(mali_c55->irqnum, NULL, mali_c55_isr,
789				   IRQF_ONESHOT, dev_driver_string(dev), dev);
790	if (ret) {
791		__mali_c55_power_off(mali_c55);
792		dev_err(dev, "failed to request irq\n");
793	}
794
795	return ret;
796}
797
798static const struct dev_pm_ops mali_c55_pm_ops = {
799	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
800				pm_runtime_force_resume)
801	SET_RUNTIME_PM_OPS(mali_c55_runtime_suspend, mali_c55_runtime_resume,
802			   NULL)
803};
804
805static int mali_c55_probe(struct platform_device *pdev)
806{
807	struct device *dev = &pdev->dev;
808	struct mali_c55 *mali_c55;
809	struct resource *res;
810	int ret;
811
812	mali_c55 = devm_kzalloc(dev, sizeof(*mali_c55), GFP_KERNEL);
813	if (!mali_c55)
814		return -ENOMEM;
815
816	mali_c55->dev = dev;
817	platform_set_drvdata(pdev, mali_c55);
818
819	mali_c55->base = devm_platform_get_and_ioremap_resource(pdev, 0,
820								&res);
821	if (IS_ERR(mali_c55->base))
822		return dev_err_probe(dev, PTR_ERR(mali_c55->base),
823				     "failed to map IO memory\n");
824
825	for (unsigned int i = 0; i < ARRAY_SIZE(mali_c55_clk_names); i++)
826		mali_c55->clks[i].id = mali_c55_clk_names[i];
827
828	ret = devm_clk_bulk_get(dev, ARRAY_SIZE(mali_c55->clks), mali_c55->clks);
829	if (ret)
830		return dev_err_probe(dev, ret, "failed to acquire clocks\n");
831
832	for (unsigned int i = 0; i < ARRAY_SIZE(mali_c55_reset_names); i++)
833		mali_c55->resets[i].id = mali_c55_reset_names[i];
834
835	ret = devm_reset_control_bulk_get_optional_shared(dev,
836			ARRAY_SIZE(mali_c55_reset_names), mali_c55->resets);
837	if (ret)
838		return dev_err_probe(dev, ret, "failed to acquire resets\n");
839
840	of_reserved_mem_device_init(dev);
841	vb2_dma_contig_set_max_seg_size(dev, UINT_MAX);
842
843	ret = __mali_c55_power_on(mali_c55);
844	if (ret)
845		return dev_err_probe(dev, ret, "failed to power on\n");
846
847	ret = mali_c55_check_hwcfg(mali_c55);
848	if (ret)
849		goto err_power_off;
850
851	ret = mali_c55_init_context(mali_c55, res);
852	if (ret)
853		goto err_power_off;
854
855	mali_c55->media_dev.dev = dev;
856
857	pm_runtime_set_autosuspend_delay(&pdev->dev, 2000);
858	pm_runtime_use_autosuspend(&pdev->dev);
859	pm_runtime_set_active(&pdev->dev);
860	pm_runtime_enable(&pdev->dev);
861
862	ret = mali_c55_media_frameworks_init(mali_c55);
863	if (ret)
864		goto err_free_context_registers;
865
866	pm_runtime_idle(&pdev->dev);
867
868	mali_c55->irqnum = platform_get_irq(pdev, 0);
869	if (mali_c55->irqnum < 0) {
870		ret = mali_c55->irqnum;
871		dev_err(dev, "failed to get interrupt\n");
872		goto err_deinit_media_frameworks;
873	}
874
875	return 0;
876
877err_deinit_media_frameworks:
878	mali_c55_media_frameworks_deinit(mali_c55);
879	pm_runtime_disable(&pdev->dev);
880err_free_context_registers:
881	kfree(mali_c55->context.registers);
882err_power_off:
883	__mali_c55_power_off(mali_c55);
884
885	return ret;
886}
887
888static void mali_c55_remove(struct platform_device *pdev)
889{
890	struct mali_c55 *mali_c55 = platform_get_drvdata(pdev);
891
892	kfree(mali_c55->context.registers);
893	mali_c55_media_frameworks_deinit(mali_c55);
894}
895
896static const struct of_device_id mali_c55_of_match[] = {
897	{ .compatible = "arm,mali-c55", },
898	{ /* Sentinel */ },
899};
900MODULE_DEVICE_TABLE(of, mali_c55_of_match);
901
902static struct platform_driver mali_c55_driver = {
903	.driver = {
904		.name = "mali-c55",
905		.of_match_table = mali_c55_of_match,
906		.pm = &mali_c55_pm_ops,
907	},
908	.probe = mali_c55_probe,
909	.remove = mali_c55_remove,
910};
911
912module_platform_driver(mali_c55_driver);
913
914MODULE_AUTHOR("Daniel Scally <dan.scally@ideasonboard.com>");
915MODULE_AUTHOR("Jacopo Mondi <jacopo.mondi@ideasonboard.com>");
916MODULE_DESCRIPTION("ARM Mali-C55 ISP platform driver");
917MODULE_LICENSE("GPL");