rpmh.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
  4 */
  5
  6#include <linux/atomic.h>
  7#include <linux/bug.h>
  8#include <linux/interrupt.h>
  9#include <linux/jiffies.h>
 10#include <linux/kernel.h>
 11#include <linux/list.h>
 12#include <linux/lockdep.h>
 13#include <linux/module.h>
 14#include <linux/of.h>
 15#include <linux/platform_device.h>
 16#include <linux/slab.h>
 17#include <linux/spinlock.h>
 18#include <linux/types.h>
 19#include <linux/wait.h>
 20
 21#include <soc/qcom/rpmh.h>
 22
 23#include "rpmh-internal.h"
 24
 25#define RPMH_TIMEOUT_MS			msecs_to_jiffies(10000)
 26
 27#define DEFINE_RPMH_MSG_ONSTACK(device, s, q, name)	\
 28	struct rpmh_request name = {			\
 29		.msg = {				\
 30			.state = s,			\
 31			.cmds = name.cmd,		\
 32			.num_cmds = 0,			\
 33			.wait_for_compl = true,		\
 34		},					\
 35		.cmd = { { 0 } },			\
 36		.completion = q,			\
 37		.dev = device,				\
 38		.needs_free = false,				\
 39	}
 40
 41#define ctrlr_to_drv(ctrlr) container_of(ctrlr, struct rsc_drv, client)
 42
 43/**
 44 * struct cache_req: the request object for caching
 45 *
 46 * @addr: the address of the resource
 47 * @sleep_val: the sleep vote
 48 * @wake_val: the wake vote
 49 * @list: linked list obj
 50 */
 51struct cache_req {
 52	u32 addr;
 53	u32 sleep_val;
 54	u32 wake_val;
 55	struct list_head list;
 56};
 57
 58/**
 59 * struct batch_cache_req - An entry in our batch catch
 60 *
 61 * @list: linked list obj
 62 * @count: number of messages
 63 * @rpm_msgs: the messages
 64 */
 65
 66struct batch_cache_req {
 67	struct list_head list;
 68	int count;
 69	struct rpmh_request rpm_msgs[];
 70};
 71
 72static struct rpmh_ctrlr *get_rpmh_ctrlr(const struct device *dev)
 73{
 74	struct rsc_drv *drv = dev_get_drvdata(dev->parent);
 75
 76	return &drv->client;
 77}
 78
 79void rpmh_tx_done(const struct tcs_request *msg)
 80{
 81	struct rpmh_request *rpm_msg = container_of(msg, struct rpmh_request,
 82						    msg);
 83	struct completion *compl = rpm_msg->completion;
 84	bool free = rpm_msg->needs_free;
 85
 86	if (!compl)
 87		goto exit;
 88
 89	/* Signal the blocking thread we are done */
 90	complete(compl);
 91
 92exit:
 93	if (free)
 94		kfree(rpm_msg);
 95}
 96
 97static struct cache_req *__find_req(struct rpmh_ctrlr *ctrlr, u32 addr)
 98{
 99	struct cache_req *p, *req = NULL;
100
101	list_for_each_entry(p, &ctrlr->cache, list) {
102		if (p->addr == addr) {
103			req = p;
104			break;
105		}
106	}
107
108	return req;
109}
110
111static struct cache_req *cache_rpm_request(struct rpmh_ctrlr *ctrlr,
112					   enum rpmh_state state,
113					   struct tcs_cmd *cmd)
114{
115	struct cache_req *req;
116	unsigned long flags;
117	u32 old_sleep_val, old_wake_val;
118
119	spin_lock_irqsave(&ctrlr->cache_lock, flags);
120	req = __find_req(ctrlr, cmd->addr);
121	if (req)
122		goto existing;
123
124	req = kzalloc(sizeof(*req), GFP_ATOMIC);
125	if (!req) {
126		req = ERR_PTR(-ENOMEM);
127		goto unlock;
128	}
129
130	req->addr = cmd->addr;
131	req->sleep_val = req->wake_val = UINT_MAX;
132	list_add_tail(&req->list, &ctrlr->cache);
133
134existing:
135	old_sleep_val = req->sleep_val;
136	old_wake_val = req->wake_val;
137
138	switch (state) {
139	case RPMH_ACTIVE_ONLY_STATE:
140	case RPMH_WAKE_ONLY_STATE:
141		req->wake_val = cmd->data;
142		break;
143	case RPMH_SLEEP_STATE:
144		req->sleep_val = cmd->data;
145		break;
146	}
147
148	ctrlr->dirty |= (req->sleep_val != old_sleep_val ||
149			 req->wake_val != old_wake_val) &&
150			 req->sleep_val != UINT_MAX &&
151			 req->wake_val != UINT_MAX;
152
153unlock:
154	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
155
156	return req;
157}
158
159/**
160 * __rpmh_write: Cache and send the RPMH request
161 *
162 * @dev: The device making the request
163 * @state: Active/Sleep request type
164 * @rpm_msg: The data that needs to be sent (cmds).
165 *
166 * Cache the RPMH request and send if the state is ACTIVE_ONLY.
167 * SLEEP/WAKE_ONLY requests are not sent to the controller at
168 * this time. Use rpmh_flush() to send them to the controller.
169 */
170static int __rpmh_write(const struct device *dev, enum rpmh_state state,
171			struct rpmh_request *rpm_msg)
172{
173	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
174	int ret = -EINVAL;
175	struct cache_req *req;
176	int i;
177
178	/* Cache the request in our store and link the payload */
179	for (i = 0; i < rpm_msg->msg.num_cmds; i++) {
180		req = cache_rpm_request(ctrlr, state, &rpm_msg->msg.cmds[i]);
181		if (IS_ERR(req))
182			return PTR_ERR(req);
183	}
184
185	if (state == RPMH_ACTIVE_ONLY_STATE) {
186		ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msg->msg);
187	} else {
188		/* Clean up our call by spoofing tx_done */
189		ret = 0;
190		rpmh_tx_done(&rpm_msg->msg);
191	}
192
193	return ret;
194}
195
196static int __fill_rpmh_msg(struct rpmh_request *req, enum rpmh_state state,
197		const struct tcs_cmd *cmd, u32 n)
198{
199	if (!cmd || !n || n > MAX_RPMH_PAYLOAD)
200		return -EINVAL;
201
202	memcpy(req->cmd, cmd, n * sizeof(*cmd));
203
204	req->msg.state = state;
205	req->msg.cmds = req->cmd;
206	req->msg.num_cmds = n;
207
208	return 0;
209}
210
211/**
212 * rpmh_write_async: Write a set of RPMH commands
213 *
214 * @dev: The device making the request
215 * @state: Active/sleep set
216 * @cmd: The payload data
217 * @n: The number of elements in payload
218 *
219 * Write a set of RPMH commands, the order of commands is maintained
220 * and will be sent as a single shot.
221 */
222int rpmh_write_async(const struct device *dev, enum rpmh_state state,
223		     const struct tcs_cmd *cmd, u32 n)
224{
225	struct rpmh_request *rpm_msg;
226	int ret;
227
228	rpm_msg = kzalloc(sizeof(*rpm_msg), GFP_ATOMIC);
229	if (!rpm_msg)
230		return -ENOMEM;
231	rpm_msg->needs_free = true;
232
233	ret = __fill_rpmh_msg(rpm_msg, state, cmd, n);
234	if (ret) {
235		kfree(rpm_msg);
236		return ret;
237	}
238
239	return __rpmh_write(dev, state, rpm_msg);
240}
241EXPORT_SYMBOL_GPL(rpmh_write_async);
242
243/**
244 * rpmh_write: Write a set of RPMH commands and block until response
245 *
246 * @dev: The device making the request
247 * @state: Active/sleep set
248 * @cmd: The payload data
249 * @n: The number of elements in @cmd
250 *
251 * May sleep. Do not call from atomic contexts.
252 */
253int rpmh_write(const struct device *dev, enum rpmh_state state,
254	       const struct tcs_cmd *cmd, u32 n)
255{
256	DECLARE_COMPLETION_ONSTACK(compl);
257	DEFINE_RPMH_MSG_ONSTACK(dev, state, &compl, rpm_msg);
258	int ret;
259
260	ret = __fill_rpmh_msg(&rpm_msg, state, cmd, n);
261	if (ret)
262		return ret;
263
264	ret = __rpmh_write(dev, state, &rpm_msg);
265	if (ret)
266		return ret;
267
268	ret = wait_for_completion_timeout(&compl, RPMH_TIMEOUT_MS);
269	WARN_ON(!ret);
270	return (ret > 0) ? 0 : -ETIMEDOUT;
271}
272EXPORT_SYMBOL_GPL(rpmh_write);
273
274static void cache_batch(struct rpmh_ctrlr *ctrlr, struct batch_cache_req *req)
275{
276	unsigned long flags;
277
278	spin_lock_irqsave(&ctrlr->cache_lock, flags);
279	list_add_tail(&req->list, &ctrlr->batch_cache);
280	ctrlr->dirty = true;
281	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
282}
283
284static int flush_batch(struct rpmh_ctrlr *ctrlr)
285{
286	struct batch_cache_req *req;
287	const struct rpmh_request *rpm_msg;
288	int ret = 0;
289	int i;
290
291	/* Send Sleep/Wake requests to the controller, expect no response */
292	list_for_each_entry(req, &ctrlr->batch_cache, list) {
293		for (i = 0; i < req->count; i++) {
294			rpm_msg = req->rpm_msgs + i;
295			ret = rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr),
296						       &rpm_msg->msg);
297			if (ret)
298				break;
299		}
300	}
301
302	return ret;
303}
304
305/**
306 * rpmh_write_batch: Write multiple sets of RPMH commands and wait for the
307 * batch to finish.
308 *
309 * @dev: the device making the request
310 * @state: Active/sleep set
311 * @cmd: The payload data
312 * @n: The array of count of elements in each batch, 0 terminated.
313 *
314 * Write a request to the RSC controller without caching. If the request
315 * state is ACTIVE, then the requests are treated as completion request
316 * and sent to the controller immediately. The function waits until all the
317 * commands are complete. If the request was to SLEEP or WAKE_ONLY, then the
318 * request is sent as fire-n-forget and no ack is expected.
319 *
320 * May sleep. Do not call from atomic contexts for ACTIVE_ONLY requests.
321 */
322int rpmh_write_batch(const struct device *dev, enum rpmh_state state,
323		     const struct tcs_cmd *cmd, u32 *n)
324{
325	struct batch_cache_req *req;
326	struct rpmh_request *rpm_msgs;
327	struct completion *compls;
328	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
329	unsigned long time_left;
330	int count = 0;
331	int ret, i;
332	void *ptr;
333
334	if (!cmd || !n)
335		return -EINVAL;
336
337	while (n[count] > 0)
338		count++;
339	if (!count)
340		return -EINVAL;
341
342	ptr = kzalloc(sizeof(*req) +
343		      count * (sizeof(req->rpm_msgs[0]) + sizeof(*compls)),
344		      GFP_ATOMIC);
345	if (!ptr)
346		return -ENOMEM;
347
348	req = ptr;
349	compls = ptr + sizeof(*req) + count * sizeof(*rpm_msgs);
350
351	req->count = count;
352	rpm_msgs = req->rpm_msgs;
353
354	for (i = 0; i < count; i++) {
355		__fill_rpmh_msg(rpm_msgs + i, state, cmd, n[i]);
356		cmd += n[i];
357	}
358
359	if (state != RPMH_ACTIVE_ONLY_STATE) {
360		cache_batch(ctrlr, req);
361		return 0;
362	}
363
364	for (i = 0; i < count; i++) {
365		struct completion *compl = &compls[i];
366
367		init_completion(compl);
368		rpm_msgs[i].completion = compl;
369		ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msgs[i].msg);
370		if (ret) {
371			pr_err("Error(%d) sending RPMH message addr=%#x\n",
372			       ret, rpm_msgs[i].msg.cmds[0].addr);
373			break;
374		}
375	}
376
377	time_left = RPMH_TIMEOUT_MS;
378	while (i--) {
379		time_left = wait_for_completion_timeout(&compls[i], time_left);
380		if (!time_left) {
381			/*
382			 * Better hope they never finish because they'll signal
383			 * the completion that we're going to free once
384			 * we've returned from this function.
385			 */
386			WARN_ON(1);
387			ret = -ETIMEDOUT;
388			goto exit;
389		}
390	}
391
392exit:
393	kfree(ptr);
394
395	return ret;
396}
397EXPORT_SYMBOL_GPL(rpmh_write_batch);
398
399static int is_req_valid(struct cache_req *req)
400{
401	return (req->sleep_val != UINT_MAX &&
402		req->wake_val != UINT_MAX &&
403		req->sleep_val != req->wake_val);
404}
405
406static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
407		       u32 addr, u32 data)
408{
409	DEFINE_RPMH_MSG_ONSTACK(NULL, state, NULL, rpm_msg);
410
411	/* Wake sets are always complete and sleep sets are not */
412	rpm_msg.msg.wait_for_compl = (state == RPMH_WAKE_ONLY_STATE);
413	rpm_msg.cmd[0].addr = addr;
414	rpm_msg.cmd[0].data = data;
415	rpm_msg.msg.num_cmds = 1;
416
417	return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg);
418}
419
420/**
421 * rpmh_flush() - Flushes the buffered sleep and wake sets to TCSes
422 *
423 * @ctrlr: Controller making request to flush cached data
424 *
425 * Return:
426 * * 0          - Success
427 * * Error code - Otherwise
428 */
429int rpmh_flush(struct rpmh_ctrlr *ctrlr)
430{
431	struct cache_req *p;
432	int ret = 0;
433
434	lockdep_assert_irqs_disabled();
435
436	/*
437	 * Currently rpmh_flush() is only called when we think we're running
438	 * on the last processor.  If the lock is busy it means another
439	 * processor is up and it's better to abort than spin.
440	 */
441	if (!spin_trylock(&ctrlr->cache_lock))
442		return -EBUSY;
443
444	if (!ctrlr->dirty) {
445		pr_debug("Skipping flush, TCS has latest data.\n");
446		goto write_next_wakeup;
447	}
448
449	/* Invalidate the TCSes first to avoid stale data */
450	rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr));
451
452	/* First flush the cached batch requests */
453	ret = flush_batch(ctrlr);
454	if (ret)
455		goto exit;
456
457	list_for_each_entry(p, &ctrlr->cache, list) {
458		if (!is_req_valid(p)) {
459			pr_debug("%s: skipping RPMH req: a:%#x s:%#x w:%#x",
460				 __func__, p->addr, p->sleep_val, p->wake_val);
461			continue;
462		}
463		ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr,
464				  p->sleep_val);
465		if (ret)
466			goto exit;
467		ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr,
468				  p->wake_val);
469		if (ret)
470			goto exit;
471	}
472
473	ctrlr->dirty = false;
474
475write_next_wakeup:
476	rpmh_rsc_write_next_wakeup(ctrlr_to_drv(ctrlr));
477exit:
478	spin_unlock(&ctrlr->cache_lock);
479	return ret;
480}
481
482/**
483 * rpmh_invalidate: Invalidate sleep and wake sets in batch_cache
484 *
485 * @dev: The device making the request
486 *
487 * Invalidate the sleep and wake values in batch_cache.
488 */
489void rpmh_invalidate(const struct device *dev)
490{
491	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
492	struct batch_cache_req *req, *tmp;
493	unsigned long flags;
494
495	spin_lock_irqsave(&ctrlr->cache_lock, flags);
496	list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list)
497		kfree(req);
498	INIT_LIST_HEAD(&ctrlr->batch_cache);
499	ctrlr->dirty = true;
500	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
501}
502EXPORT_SYMBOL_GPL(rpmh_invalidate);