main.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright(c) Advanced Micro Devices, Inc */
  3
  4#include <linux/module.h>
  5#include <linux/auxiliary_bus.h>
  6#include <linux/pci.h>
  7#include <linux/vmalloc.h>
  8#include <linux/bitfield.h>
  9#include <linux/string.h>
 10
 11#include <uapi/fwctl/fwctl.h>
 12#include <uapi/fwctl/pds.h>
 13#include <linux/fwctl.h>
 14
 15#include <linux/pds/pds_common.h>
 16#include <linux/pds/pds_core_if.h>
 17#include <linux/pds/pds_adminq.h>
 18#include <linux/pds/pds_auxbus.h>
 19
 20struct pdsfc_uctx {
 21	struct fwctl_uctx uctx;
 22	u32 uctx_caps;
 23};
 24
 25struct pdsfc_rpc_endpoint_info {
 26	u32 endpoint;
 27	dma_addr_t operations_pa;
 28	struct pds_fwctl_query_data *operations;
 29	struct mutex lock;	/* lock for endpoint info management */
 30};
 31
 32struct pdsfc_dev {
 33	struct fwctl_device fwctl;
 34	struct pds_auxiliary_dev *padev;
 35	u32 caps;
 36	struct pds_fwctl_ident ident;
 37	dma_addr_t endpoints_pa;
 38	struct pds_fwctl_query_data *endpoints;
 39	struct pdsfc_rpc_endpoint_info *endpoint_info;
 40};
 41
 42static int pdsfc_open_uctx(struct fwctl_uctx *uctx)
 43{
 44	struct pdsfc_dev *pdsfc = container_of(uctx->fwctl, struct pdsfc_dev, fwctl);
 45	struct pdsfc_uctx *pdsfc_uctx = container_of(uctx, struct pdsfc_uctx, uctx);
 46
 47	pdsfc_uctx->uctx_caps = pdsfc->caps;
 48
 49	return 0;
 50}
 51
 52static void pdsfc_close_uctx(struct fwctl_uctx *uctx)
 53{
 54}
 55
 56static void *pdsfc_info(struct fwctl_uctx *uctx, size_t *length)
 57{
 58	struct pdsfc_uctx *pdsfc_uctx = container_of(uctx, struct pdsfc_uctx, uctx);
 59	struct fwctl_info_pds *info;
 60
 61	info = kzalloc(sizeof(*info), GFP_KERNEL);
 62	if (!info)
 63		return ERR_PTR(-ENOMEM);
 64
 65	info->uctx_caps = pdsfc_uctx->uctx_caps;
 66
 67	return info;
 68}
 69
 70static int pdsfc_identify(struct pdsfc_dev *pdsfc)
 71{
 72	struct device *dev = &pdsfc->fwctl.dev;
 73	union pds_core_adminq_comp comp = {0};
 74	union pds_core_adminq_cmd cmd;
 75	struct pds_fwctl_ident *ident;
 76	dma_addr_t ident_pa;
 77	int err;
 78
 79	ident = dma_alloc_coherent(dev->parent, sizeof(*ident), &ident_pa, GFP_KERNEL);
 80	if (!ident) {
 81		dev_err(dev, "Failed to map ident buffer\n");
 82		return -ENOMEM;
 83	}
 84
 85	cmd = (union pds_core_adminq_cmd) {
 86		.fwctl_ident = {
 87			.opcode = PDS_FWCTL_CMD_IDENT,
 88			.version = 0,
 89			.len = cpu_to_le32(sizeof(*ident)),
 90			.ident_pa = cpu_to_le64(ident_pa),
 91		}
 92	};
 93
 94	err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0);
 95	if (err)
 96		dev_err(dev, "Failed to send adminq cmd opcode: %u err: %d\n",
 97			cmd.fwctl_ident.opcode, err);
 98	else
 99		pdsfc->ident = *ident;
100
101	dma_free_coherent(dev->parent, sizeof(*ident), ident, ident_pa);
102
103	return err;
104}
105
106static void pdsfc_free_endpoints(struct pdsfc_dev *pdsfc)
107{
108	struct device *dev = &pdsfc->fwctl.dev;
109	u32 num_endpoints;
110	int i;
111
112	if (!pdsfc->endpoints)
113		return;
114
115	num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries);
116	for (i = 0; pdsfc->endpoint_info && i < num_endpoints; i++)
117		mutex_destroy(&pdsfc->endpoint_info[i].lock);
118	vfree(pdsfc->endpoint_info);
119	pdsfc->endpoint_info = NULL;
120	dma_free_coherent(dev->parent, PAGE_SIZE,
121			  pdsfc->endpoints, pdsfc->endpoints_pa);
122	pdsfc->endpoints = NULL;
123	pdsfc->endpoints_pa = DMA_MAPPING_ERROR;
124}
125
126static void pdsfc_free_operations(struct pdsfc_dev *pdsfc)
127{
128	struct device *dev = &pdsfc->fwctl.dev;
129	u32 num_endpoints;
130	int i;
131
132	num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries);
133	for (i = 0; i < num_endpoints; i++) {
134		struct pdsfc_rpc_endpoint_info *ei = &pdsfc->endpoint_info[i];
135
136		if (ei->operations) {
137			dma_free_coherent(dev->parent, PAGE_SIZE,
138					  ei->operations, ei->operations_pa);
139			ei->operations = NULL;
140			ei->operations_pa = DMA_MAPPING_ERROR;
141		}
142	}
143}
144
145static struct pds_fwctl_query_data *pdsfc_get_endpoints(struct pdsfc_dev *pdsfc,
146							dma_addr_t *pa)
147{
148	struct device *dev = &pdsfc->fwctl.dev;
149	union pds_core_adminq_comp comp = {0};
150	struct pds_fwctl_query_data *data;
151	union pds_core_adminq_cmd cmd;
152	dma_addr_t data_pa;
153	int err;
154
155	data = dma_alloc_coherent(dev->parent, PAGE_SIZE, &data_pa, GFP_KERNEL);
156	if (!data) {
157		dev_err(dev, "Failed to map endpoint list\n");
158		return ERR_PTR(-ENOMEM);
159	}
160
161	cmd = (union pds_core_adminq_cmd) {
162		.fwctl_query = {
163			.opcode = PDS_FWCTL_CMD_QUERY,
164			.entity = PDS_FWCTL_RPC_ROOT,
165			.version = 0,
166			.query_data_buf_len = cpu_to_le32(PAGE_SIZE),
167			.query_data_buf_pa = cpu_to_le64(data_pa),
168		}
169	};
170
171	err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0);
172	if (err) {
173		dev_err(dev, "Failed to send adminq cmd opcode: %u entity: %u err: %d\n",
174			cmd.fwctl_query.opcode, cmd.fwctl_query.entity, err);
175		dma_free_coherent(dev->parent, PAGE_SIZE, data, data_pa);
176		return ERR_PTR(err);
177	}
178
179	*pa = data_pa;
180
181	return data;
182}
183
184static int pdsfc_init_endpoints(struct pdsfc_dev *pdsfc)
185{
186	struct pds_fwctl_query_data_endpoint *ep_entry;
187	u32 num_endpoints;
188	int i;
189
190	pdsfc->endpoints = pdsfc_get_endpoints(pdsfc, &pdsfc->endpoints_pa);
191	if (IS_ERR(pdsfc->endpoints))
192		return PTR_ERR(pdsfc->endpoints);
193
194	num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries);
195	pdsfc->endpoint_info = vcalloc(num_endpoints,
196				       sizeof(*pdsfc->endpoint_info));
197	if (!pdsfc->endpoint_info) {
198		pdsfc_free_endpoints(pdsfc);
199		return -ENOMEM;
200	}
201
202	ep_entry = (struct pds_fwctl_query_data_endpoint *)pdsfc->endpoints->entries;
203	for (i = 0; i < num_endpoints; i++) {
204		mutex_init(&pdsfc->endpoint_info[i].lock);
205		pdsfc->endpoint_info[i].endpoint = le32_to_cpu(ep_entry[i].id);
206	}
207
208	return 0;
209}
210
211static struct pds_fwctl_query_data *pdsfc_get_operations(struct pdsfc_dev *pdsfc,
212							 dma_addr_t *pa, u32 ep)
213{
214	struct pds_fwctl_query_data_operation *entries;
215	struct device *dev = &pdsfc->fwctl.dev;
216	union pds_core_adminq_comp comp = {0};
217	struct pds_fwctl_query_data *data;
218	union pds_core_adminq_cmd cmd;
219	dma_addr_t data_pa;
220	u32 num_entries;
221	int err;
222	int i;
223
224	/* Query the operations list for the given endpoint */
225	data = dma_alloc_coherent(dev->parent, PAGE_SIZE, &data_pa, GFP_KERNEL);
226	if (!data) {
227		dev_err(dev, "Failed to map operations list\n");
228		return ERR_PTR(-ENOMEM);
229	}
230
231	cmd = (union pds_core_adminq_cmd) {
232		.fwctl_query = {
233			.opcode = PDS_FWCTL_CMD_QUERY,
234			.entity = PDS_FWCTL_RPC_ENDPOINT,
235			.version = 0,
236			.query_data_buf_len = cpu_to_le32(PAGE_SIZE),
237			.query_data_buf_pa = cpu_to_le64(data_pa),
238			.ep = cpu_to_le32(ep),
239		}
240	};
241
242	err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0);
243	if (err) {
244		dev_err(dev, "Failed to send adminq cmd opcode: %u entity: %u err: %d\n",
245			cmd.fwctl_query.opcode, cmd.fwctl_query.entity, err);
246		dma_free_coherent(dev->parent, PAGE_SIZE, data, data_pa);
247		return ERR_PTR(err);
248	}
249
250	*pa = data_pa;
251
252	entries = (struct pds_fwctl_query_data_operation *)data->entries;
253	num_entries = le32_to_cpu(data->num_entries);
254	dev_dbg(dev, "num_entries %d\n", num_entries);
255	for (i = 0; i < num_entries; i++) {
256
257		/* Translate FW command attribute to fwctl scope */
258		switch (entries[i].scope) {
259		case PDSFC_FW_CMD_ATTR_READ:
260		case PDSFC_FW_CMD_ATTR_WRITE:
261		case PDSFC_FW_CMD_ATTR_SYNC:
262			entries[i].scope = FWCTL_RPC_CONFIGURATION;
263			break;
264		case PDSFC_FW_CMD_ATTR_DEBUG_READ:
265			entries[i].scope = FWCTL_RPC_DEBUG_READ_ONLY;
266			break;
267		case PDSFC_FW_CMD_ATTR_DEBUG_WRITE:
268			entries[i].scope = FWCTL_RPC_DEBUG_WRITE;
269			break;
270		default:
271			entries[i].scope = FWCTL_RPC_DEBUG_WRITE_FULL;
272			break;
273		}
274		dev_dbg(dev, "endpoint %d operation: id %x scope %d\n",
275			ep, le32_to_cpu(entries[i].id), entries[i].scope);
276	}
277
278	return data;
279}
280
281static int pdsfc_validate_rpc(struct pdsfc_dev *pdsfc,
282			      struct fwctl_rpc_pds *rpc,
283			      enum fwctl_rpc_scope scope)
284{
285	struct pds_fwctl_query_data_operation *op_entry;
286	struct pdsfc_rpc_endpoint_info *ep_info = NULL;
287	struct device *dev = &pdsfc->fwctl.dev;
288	u32 num_entries;
289	int i;
290
291	/* validate rpc in_len & out_len based
292	 * on ident.max_req_sz & max_resp_sz
293	 */
294	if (rpc->in.len > le32_to_cpu(pdsfc->ident.max_req_sz)) {
295		dev_dbg(dev, "Invalid request size %u, max %u\n",
296			rpc->in.len, le32_to_cpu(pdsfc->ident.max_req_sz));
297		return -EINVAL;
298	}
299
300	if (rpc->out.len > le32_to_cpu(pdsfc->ident.max_resp_sz)) {
301		dev_dbg(dev, "Invalid response size %u, max %u\n",
302			rpc->out.len, le32_to_cpu(pdsfc->ident.max_resp_sz));
303		return -EINVAL;
304	}
305
306	num_entries = le32_to_cpu(pdsfc->endpoints->num_entries);
307	for (i = 0; i < num_entries; i++) {
308		if (pdsfc->endpoint_info[i].endpoint == rpc->in.ep) {
309			ep_info = &pdsfc->endpoint_info[i];
310			break;
311		}
312	}
313	if (!ep_info) {
314		dev_dbg(dev, "Invalid endpoint %d\n", rpc->in.ep);
315		return -EINVAL;
316	}
317
318	/* query and cache this endpoint's operations */
319	mutex_lock(&ep_info->lock);
320	if (!ep_info->operations) {
321		struct pds_fwctl_query_data *operations;
322
323		operations = pdsfc_get_operations(pdsfc,
324						  &ep_info->operations_pa,
325						  rpc->in.ep);
326		if (IS_ERR(operations)) {
327			mutex_unlock(&ep_info->lock);
328			return -ENOMEM;
329		}
330		ep_info->operations = operations;
331	}
332	mutex_unlock(&ep_info->lock);
333
334	/* reject unsupported and/or out of scope commands */
335	op_entry = (struct pds_fwctl_query_data_operation *)ep_info->operations->entries;
336	num_entries = le32_to_cpu(ep_info->operations->num_entries);
337	for (i = 0; i < num_entries; i++) {
338		if (PDS_FWCTL_RPC_OPCODE_CMP(rpc->in.op, le32_to_cpu(op_entry[i].id))) {
339			if (scope < op_entry[i].scope)
340				return -EPERM;
341			return 0;
342		}
343	}
344
345	dev_dbg(dev, "Invalid operation %d for endpoint %d\n", rpc->in.op, rpc->in.ep);
346
347	return -EINVAL;
348}
349
350static void *pdsfc_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
351			  void *in, size_t in_len, size_t *out_len)
352{
353	struct pdsfc_dev *pdsfc = container_of(uctx->fwctl, struct pdsfc_dev, fwctl);
354	struct device *dev = &uctx->fwctl->dev;
355	union pds_core_adminq_comp comp = {0};
356	dma_addr_t out_payload_dma_addr = 0;
357	dma_addr_t in_payload_dma_addr = 0;
358	struct fwctl_rpc_pds *rpc = in;
359	union pds_core_adminq_cmd cmd;
360	void *out_payload = NULL;
361	void *in_payload = NULL;
362	void *out = NULL;
363	int err;
364
365	err = pdsfc_validate_rpc(pdsfc, rpc, scope);
366	if (err)
367		return ERR_PTR(err);
368
369	if (rpc->in.len > 0) {
370		in_payload = memdup_user(u64_to_user_ptr(rpc->in.payload), rpc->in.len);
371		if (IS_ERR(in_payload)) {
372			dev_dbg(dev, "Failed to copy in_payload from user\n");
373			return in_payload;
374		}
375
376		in_payload_dma_addr = dma_map_single(dev->parent, in_payload,
377						     rpc->in.len, DMA_TO_DEVICE);
378		err = dma_mapping_error(dev->parent, in_payload_dma_addr);
379		if (err) {
380			dev_dbg(dev, "Failed to map in_payload\n");
381			goto err_in_payload;
382		}
383	}
384
385	if (rpc->out.len > 0) {
386		out_payload = kzalloc(rpc->out.len, GFP_KERNEL);
387		if (!out_payload) {
388			dev_dbg(dev, "Failed to allocate out_payload\n");
389			err = -ENOMEM;
390			goto err_out_payload;
391		}
392
393		out_payload_dma_addr = dma_map_single(dev->parent, out_payload,
394						      rpc->out.len, DMA_FROM_DEVICE);
395		err = dma_mapping_error(dev->parent, out_payload_dma_addr);
396		if (err) {
397			dev_dbg(dev, "Failed to map out_payload\n");
398			goto err_out_payload;
399		}
400	}
401
402	cmd = (union pds_core_adminq_cmd) {
403		.fwctl_rpc = {
404			.opcode = PDS_FWCTL_CMD_RPC,
405			.flags = cpu_to_le16(PDS_FWCTL_RPC_IND_REQ | PDS_FWCTL_RPC_IND_RESP),
406			.ep = cpu_to_le32(rpc->in.ep),
407			.op = cpu_to_le32(rpc->in.op),
408			.req_pa = cpu_to_le64(in_payload_dma_addr),
409			.req_sz = cpu_to_le32(rpc->in.len),
410			.resp_pa = cpu_to_le64(out_payload_dma_addr),
411			.resp_sz = cpu_to_le32(rpc->out.len),
412		}
413	};
414
415	err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0);
416	if (err) {
417		dev_dbg(dev, "%s: ep %d op %x req_pa %llx req_sz %d req_sg %d resp_pa %llx resp_sz %d resp_sg %d err %d\n",
418			__func__, rpc->in.ep, rpc->in.op,
419			cmd.fwctl_rpc.req_pa, cmd.fwctl_rpc.req_sz, cmd.fwctl_rpc.req_sg_elems,
420			cmd.fwctl_rpc.resp_pa, cmd.fwctl_rpc.resp_sz, cmd.fwctl_rpc.resp_sg_elems,
421			err);
422		goto done;
423	}
424
425	dynamic_hex_dump("out ", DUMP_PREFIX_OFFSET, 16, 1, out_payload, rpc->out.len, true);
426
427	if (copy_to_user(u64_to_user_ptr(rpc->out.payload), out_payload, rpc->out.len)) {
428		dev_dbg(dev, "Failed to copy out_payload to user\n");
429		out = ERR_PTR(-EFAULT);
430		goto done;
431	}
432
433	rpc->out.retval = le32_to_cpu(comp.fwctl_rpc.err);
434	*out_len = in_len;
435	out = in;
436
437done:
438	if (out_payload_dma_addr)
439		dma_unmap_single(dev->parent, out_payload_dma_addr,
440				 rpc->out.len, DMA_FROM_DEVICE);
441err_out_payload:
442	kfree(out_payload);
443
444	if (in_payload_dma_addr)
445		dma_unmap_single(dev->parent, in_payload_dma_addr,
446				 rpc->in.len, DMA_TO_DEVICE);
447err_in_payload:
448	kfree(in_payload);
449	if (err)
450		return ERR_PTR(err);
451
452	return out;
453}
454
455static const struct fwctl_ops pdsfc_ops = {
456	.device_type = FWCTL_DEVICE_TYPE_PDS,
457	.uctx_size = sizeof(struct pdsfc_uctx),
458	.open_uctx = pdsfc_open_uctx,
459	.close_uctx = pdsfc_close_uctx,
460	.info = pdsfc_info,
461	.fw_rpc = pdsfc_fw_rpc,
462};
463
464static int pdsfc_probe(struct auxiliary_device *adev,
465		       const struct auxiliary_device_id *id)
466{
467	struct pds_auxiliary_dev *padev =
468			container_of(adev, struct pds_auxiliary_dev, aux_dev);
469	struct device *dev = &adev->dev;
470	struct pdsfc_dev *pdsfc;
471	int err;
472
473	pdsfc = fwctl_alloc_device(&padev->vf_pdev->dev, &pdsfc_ops,
474				   struct pdsfc_dev, fwctl);
475	if (!pdsfc)
476		return -ENOMEM;
477	pdsfc->padev = padev;
478
479	err = pdsfc_identify(pdsfc);
480	if (err) {
481		fwctl_put(&pdsfc->fwctl);
482		return dev_err_probe(dev, err, "Failed to identify device\n");
483	}
484
485	err = pdsfc_init_endpoints(pdsfc);
486	if (err) {
487		fwctl_put(&pdsfc->fwctl);
488		return dev_err_probe(dev, err, "Failed to init endpoints\n");
489	}
490
491	pdsfc->caps = PDS_FWCTL_QUERY_CAP | PDS_FWCTL_SEND_CAP;
492
493	err = fwctl_register(&pdsfc->fwctl);
494	if (err) {
495		pdsfc_free_endpoints(pdsfc);
496		fwctl_put(&pdsfc->fwctl);
497		return dev_err_probe(dev, err, "Failed to register device\n");
498	}
499
500	auxiliary_set_drvdata(adev, pdsfc);
501
502	return 0;
503}
504
505static void pdsfc_remove(struct auxiliary_device *adev)
506{
507	struct pdsfc_dev *pdsfc = auxiliary_get_drvdata(adev);
508
509	fwctl_unregister(&pdsfc->fwctl);
510	pdsfc_free_operations(pdsfc);
511	pdsfc_free_endpoints(pdsfc);
512
513	fwctl_put(&pdsfc->fwctl);
514}
515
516static const struct auxiliary_device_id pdsfc_id_table[] = {
517	{.name = PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_FWCTL_STR },
518	{}
519};
520MODULE_DEVICE_TABLE(auxiliary, pdsfc_id_table);
521
522static struct auxiliary_driver pdsfc_driver = {
523	.name = "pds_fwctl",
524	.probe = pdsfc_probe,
525	.remove = pdsfc_remove,
526	.id_table = pdsfc_id_table,
527};
528
529module_auxiliary_driver(pdsfc_driver);
530
531MODULE_IMPORT_NS("FWCTL");
532MODULE_DESCRIPTION("pds fwctl driver");
533MODULE_AUTHOR("Shannon Nelson <shannon.nelson@amd.com>");
534MODULE_AUTHOR("Brett Creeley <brett.creeley@amd.com>");
535MODULE_LICENSE("GPL");