pmem.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
  3#include <linux/libnvdimm.h>
  4#include <linux/unaligned.h>
  5#include <linux/device.h>
  6#include <linux/module.h>
  7#include <linux/ndctl.h>
  8#include <linux/async.h>
  9#include <linux/slab.h>
 10#include <linux/nd.h>
 11#include "cxlmem.h"
 12#include "cxl.h"
 13
 14static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
 15
 16/**
 17 * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology
 18 * @host: platform firmware root device
 19 * @port: CXL port at the root of a CXL topology
 20 *
 21 * Return: bridge device that can host cxl_nvdimm objects
 22 */
 23struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
 24						     struct cxl_port *port)
 25{
 26	return __devm_cxl_add_nvdimm_bridge(host, port);
 27}
 28EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL");
 29
 30static void clear_exclusive(void *mds)
 31{
 32	clear_exclusive_cxl_commands(mds, exclusive_cmds);
 33}
 34
 35static void unregister_nvdimm(void *nvdimm)
 36{
 37	nvdimm_delete(nvdimm);
 38}
 39
 40static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf)
 41{
 42	struct nvdimm *nvdimm = to_nvdimm(dev);
 43	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
 44
 45	return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
 46}
 47static DEVICE_ATTR_RO(provider);
 48
 49static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf)
 50{
 51	struct nvdimm *nvdimm = to_nvdimm(dev);
 52	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
 53	struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;
 54
 55	return sysfs_emit(buf, "%lld\n", cxlds->serial);
 56}
 57static DEVICE_ATTR_RO(id);
 58
 59static ssize_t dirty_shutdown_show(struct device *dev,
 60				   struct device_attribute *attr, char *buf)
 61{
 62	struct nvdimm *nvdimm = to_nvdimm(dev);
 63	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
 64
 65	return sysfs_emit(buf, "%llu\n", cxl_nvd->dirty_shutdowns);
 66}
 67static DEVICE_ATTR_RO(dirty_shutdown);
 68
 69static struct attribute *cxl_dimm_attributes[] = {
 70	&dev_attr_id.attr,
 71	&dev_attr_provider.attr,
 72	&dev_attr_dirty_shutdown.attr,
 73	NULL
 74};
 75
 76#define CXL_INVALID_DIRTY_SHUTDOWN_COUNT ULLONG_MAX
 77static umode_t cxl_dimm_visible(struct kobject *kobj,
 78				struct attribute *a, int n)
 79{
 80	if (a == &dev_attr_dirty_shutdown.attr) {
 81		struct device *dev = kobj_to_dev(kobj);
 82		struct nvdimm *nvdimm = to_nvdimm(dev);
 83		struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
 84
 85		if (cxl_nvd->dirty_shutdowns ==
 86		    CXL_INVALID_DIRTY_SHUTDOWN_COUNT)
 87			return 0;
 88	}
 89
 90	return a->mode;
 91}
 92
 93static const struct attribute_group cxl_dimm_attribute_group = {
 94	.name = "cxl",
 95	.attrs = cxl_dimm_attributes,
 96	.is_visible = cxl_dimm_visible
 97};
 98
 99static const struct attribute_group *cxl_dimm_attribute_groups[] = {
100	&cxl_dimm_attribute_group,
101	NULL
102};
103
104static void cxl_nvdimm_arm_dirty_shutdown_tracking(struct cxl_nvdimm *cxl_nvd)
105{
106	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
107	struct cxl_dev_state *cxlds = cxlmd->cxlds;
108	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
109	struct device *dev = &cxl_nvd->dev;
110	u32 count;
111
112	/*
113	 * Dirty tracking is enabled and exposed to the user, only when:
114	 *   - dirty shutdown on the device can be set, and,
115	 *   - the device has a Device GPF DVSEC (albeit unused), and,
116	 *   - the Get Health Info cmd can retrieve the device's dirty count.
117	 */
118	cxl_nvd->dirty_shutdowns = CXL_INVALID_DIRTY_SHUTDOWN_COUNT;
119
120	if (cxl_arm_dirty_shutdown(mds)) {
121		dev_warn(dev, "GPF: could not set dirty shutdown state\n");
122		return;
123	}
124
125	if (!cxl_gpf_get_dvsec(cxlds->dev))
126		return;
127
128	if (cxl_get_dirty_count(mds, &count)) {
129		dev_warn(dev, "GPF: could not retrieve dirty count\n");
130		return;
131	}
132
133	cxl_nvd->dirty_shutdowns = count;
134}
135
136static int cxl_nvdimm_probe(struct device *dev)
137{
138	struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
139	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
140	struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
141	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
142	unsigned long flags = 0, cmd_mask = 0;
143	struct nvdimm *nvdimm;
144	int rc;
145
146	if (test_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags))
147		return -EBUSY;
148
149	set_exclusive_cxl_commands(mds, exclusive_cmds);
150	rc = devm_add_action_or_reset(dev, clear_exclusive, mds);
151	if (rc)
152		return rc;
153
154	set_bit(NDD_LABELING, &flags);
155	set_bit(NDD_REGISTER_SYNC, &flags);
156	set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
157	set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
158	set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
159
160	/*
161	 * Set dirty shutdown now, with the expectation that the device
162	 * clear it upon a successful GPF flow. The exception to this
163	 * is upon Viral detection, per CXL 3.2 section 12.4.2.
164	 */
165	cxl_nvdimm_arm_dirty_shutdown_tracking(cxl_nvd);
166
167	nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
168				 cxl_dimm_attribute_groups, flags,
169				 cmd_mask, 0, NULL, cxl_nvd->dev_id,
170				 cxl_security_ops, NULL);
171	if (!nvdimm)
172		return -ENOMEM;
173
174	dev_set_drvdata(dev, nvdimm);
175	return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
176}
177
178static struct cxl_driver cxl_nvdimm_driver = {
179	.name = "cxl_nvdimm",
180	.probe = cxl_nvdimm_probe,
181	.id = CXL_DEVICE_NVDIMM,
182	.drv = {
183		.suppress_bind_attrs = true,
184	},
185};
186
187static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds,
188				    struct nd_cmd_get_config_size *cmd,
189				    unsigned int buf_len)
190{
191	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
192
193	if (sizeof(*cmd) > buf_len)
194		return -EINVAL;
195
196	*cmd = (struct nd_cmd_get_config_size){
197		.config_size = mds->lsa_size,
198		.max_xfer =
199			cxl_mbox->payload_size - sizeof(struct cxl_mbox_set_lsa),
200	};
201
202	return 0;
203}
204
205static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds,
206				    struct nd_cmd_get_config_data_hdr *cmd,
207				    unsigned int buf_len)
208{
209	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
210	struct cxl_mbox_get_lsa get_lsa;
211	struct cxl_mbox_cmd mbox_cmd;
212	int rc;
213
214	if (sizeof(*cmd) > buf_len)
215		return -EINVAL;
216	if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
217		return -EINVAL;
218
219	get_lsa = (struct cxl_mbox_get_lsa) {
220		.offset = cpu_to_le32(cmd->in_offset),
221		.length = cpu_to_le32(cmd->in_length),
222	};
223	mbox_cmd = (struct cxl_mbox_cmd) {
224		.opcode = CXL_MBOX_OP_GET_LSA,
225		.payload_in = &get_lsa,
226		.size_in = sizeof(get_lsa),
227		.size_out = cmd->in_length,
228		.payload_out = cmd->out_buf,
229	};
230
231	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
232	cmd->status = 0;
233
234	return rc;
235}
236
237static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds,
238				    struct nd_cmd_set_config_hdr *cmd,
239				    unsigned int buf_len)
240{
241	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
242	struct cxl_mbox_set_lsa *set_lsa;
243	struct cxl_mbox_cmd mbox_cmd;
244	int rc;
245
246	if (sizeof(*cmd) > buf_len)
247		return -EINVAL;
248
249	/* 4-byte status follows the input data in the payload */
250	if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
251		return -EINVAL;
252
253	set_lsa =
254		kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
255	if (!set_lsa)
256		return -ENOMEM;
257
258	*set_lsa = (struct cxl_mbox_set_lsa) {
259		.offset = cpu_to_le32(cmd->in_offset),
260	};
261	memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
262	mbox_cmd = (struct cxl_mbox_cmd) {
263		.opcode = CXL_MBOX_OP_SET_LSA,
264		.payload_in = set_lsa,
265		.size_in = struct_size(set_lsa, data, cmd->in_length),
266	};
267
268	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
269
270	/*
271	 * Set "firmware" status (4-packed bytes at the end of the input
272	 * payload.
273	 */
274	put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
275	kvfree(set_lsa);
276
277	return rc;
278}
279
280static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
281			       void *buf, unsigned int buf_len)
282{
283	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
284	unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
285	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
286	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
287
288	if (!test_bit(cmd, &cmd_mask))
289		return -ENOTTY;
290
291	switch (cmd) {
292	case ND_CMD_GET_CONFIG_SIZE:
293		return cxl_pmem_get_config_size(mds, buf, buf_len);
294	case ND_CMD_GET_CONFIG_DATA:
295		return cxl_pmem_get_config_data(mds, buf, buf_len);
296	case ND_CMD_SET_CONFIG_DATA:
297		return cxl_pmem_set_config_data(mds, buf, buf_len);
298	default:
299		return -ENOTTY;
300	}
301}
302
303static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
304			struct nvdimm *nvdimm, unsigned int cmd, void *buf,
305			unsigned int buf_len, int *cmd_rc)
306{
307	/*
308	 * No firmware response to translate, let the transport error
309	 * code take precedence.
310	 */
311	*cmd_rc = 0;
312
313	if (!nvdimm)
314		return -ENOTTY;
315	return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
316}
317
318static int detach_nvdimm(struct device *dev, void *data)
319{
320	struct cxl_nvdimm *cxl_nvd;
321	bool release = false;
322
323	if (!is_cxl_nvdimm(dev))
324		return 0;
325
326	scoped_guard(device, dev) {
327		if (dev->driver) {
328			cxl_nvd = to_cxl_nvdimm(dev);
329			if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) {
330				release = true;
331				set_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags);
332			}
333		}
334	}
335	if (release)
336		device_release_driver(dev);
337	return 0;
338}
339
340static void unregister_nvdimm_bus(void *_cxl_nvb)
341{
342	struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
343	struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;
344
345	bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);
346
347	cxl_nvb->nvdimm_bus = NULL;
348	nvdimm_bus_unregister(nvdimm_bus);
349}
350
351static int cxl_nvdimm_bridge_probe(struct device *dev)
352{
353	struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
354
355	cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
356		.provider_name = "CXL",
357		.module = THIS_MODULE,
358		.ndctl = cxl_pmem_ctl,
359	};
360
361	cxl_nvb->nvdimm_bus =
362		nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
363
364	if (!cxl_nvb->nvdimm_bus)
365		return -ENOMEM;
366
367	return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
368}
369
370static struct cxl_driver cxl_nvdimm_bridge_driver = {
371	.name = "cxl_nvdimm_bridge",
372	.probe = cxl_nvdimm_bridge_probe,
373	.id = CXL_DEVICE_NVDIMM_BRIDGE,
374	.drv = {
375		.probe_type = PROBE_FORCE_SYNCHRONOUS,
376		.suppress_bind_attrs = true,
377	},
378};
379
380static void unregister_nvdimm_region(void *nd_region)
381{
382	nvdimm_region_delete(nd_region);
383}
384
385static void cxlr_pmem_remove_resource(void *res)
386{
387	remove_resource(res);
388}
389
390struct cxl_pmem_region_info {
391	u64 offset;
392	u64 serial;
393};
394
395static int cxl_pmem_region_probe(struct device *dev)
396{
397	struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
398	struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
399	struct cxl_region *cxlr = cxlr_pmem->cxlr;
400	struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
401	struct cxl_pmem_region_info *info = NULL;
402	struct nd_interleave_set *nd_set;
403	struct nd_region_desc ndr_desc;
404	struct cxl_nvdimm *cxl_nvd;
405	struct nvdimm *nvdimm;
406	struct resource *res;
407	int rc, i = 0;
408
409	memset(&mappings, 0, sizeof(mappings));
410	memset(&ndr_desc, 0, sizeof(ndr_desc));
411
412	res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
413	if (!res)
414		return -ENOMEM;
415
416	res->name = "Persistent Memory";
417	res->start = cxlr_pmem->hpa_range.start;
418	res->end = cxlr_pmem->hpa_range.end;
419	res->flags = IORESOURCE_MEM;
420	res->desc = IORES_DESC_PERSISTENT_MEMORY;
421
422	rc = insert_resource(&iomem_resource, res);
423	if (rc)
424		return rc;
425
426	rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
427	if (rc)
428		return rc;
429
430	ndr_desc.res = res;
431	ndr_desc.provider_data = cxlr_pmem;
432
433	ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
434	ndr_desc.target_node = phys_to_target_node(res->start);
435	if (ndr_desc.target_node == NUMA_NO_NODE) {
436		ndr_desc.target_node = ndr_desc.numa_node;
437		dev_dbg(&cxlr->dev, "changing target node from %d to %d",
438			NUMA_NO_NODE, ndr_desc.target_node);
439	}
440
441	nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
442	if (!nd_set)
443		return -ENOMEM;
444
445	ndr_desc.memregion = cxlr->id;
446	set_bit(ND_REGION_CXL, &ndr_desc.flags);
447	set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
448
449	info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
450	if (!info)
451		return -ENOMEM;
452
453	for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
454		struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
455		struct cxl_memdev *cxlmd = m->cxlmd;
456		struct cxl_dev_state *cxlds = cxlmd->cxlds;
457
458		cxl_nvd = cxlmd->cxl_nvd;
459		nvdimm = dev_get_drvdata(&cxl_nvd->dev);
460		if (!nvdimm) {
461			dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
462				dev_name(&cxlmd->dev));
463			rc = -ENODEV;
464			goto out_nvd;
465		}
466
467		if (cxlds->serial == 0) {
468			/* include missing alongside invalid in this error message. */
469			dev_err(dev, "%s: invalid or missing serial number\n",
470				dev_name(&cxlmd->dev));
471			rc = -ENXIO;
472			goto out_nvd;
473		}
474		info[i].serial = cxlds->serial;
475		info[i].offset = m->start;
476
477		m->cxl_nvd = cxl_nvd;
478		mappings[i] = (struct nd_mapping_desc) {
479			.nvdimm = nvdimm,
480			.start = m->start,
481			.size = m->size,
482			.position = i,
483		};
484	}
485	ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
486	ndr_desc.mapping = mappings;
487
488	/*
489	 * TODO enable CXL labels which skip the need for 'interleave-set cookie'
490	 */
491	nd_set->cookie1 =
492		nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
493	nd_set->cookie2 = nd_set->cookie1;
494	ndr_desc.nd_set = nd_set;
495
496	cxlr_pmem->nd_region =
497		nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
498	if (!cxlr_pmem->nd_region) {
499		rc = -ENOMEM;
500		goto out_nvd;
501	}
502
503	rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
504				      cxlr_pmem->nd_region);
505out_nvd:
506	kfree(info);
507
508	return rc;
509}
510
511static struct cxl_driver cxl_pmem_region_driver = {
512	.name = "cxl_pmem_region",
513	.probe = cxl_pmem_region_probe,
514	.id = CXL_DEVICE_PMEM_REGION,
515	.drv = {
516		.suppress_bind_attrs = true,
517	},
518};
519
520static __init int cxl_pmem_init(void)
521{
522	int rc;
523
524	set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
525	set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
526
527	rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
528	if (rc)
529		return rc;
530
531	rc = cxl_driver_register(&cxl_nvdimm_driver);
532	if (rc)
533		goto err_nvdimm;
534
535	rc = cxl_driver_register(&cxl_pmem_region_driver);
536	if (rc)
537		goto err_region;
538
539	return 0;
540
541err_region:
542	cxl_driver_unregister(&cxl_nvdimm_driver);
543err_nvdimm:
544	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
545	return rc;
546}
547
548static __exit void cxl_pmem_exit(void)
549{
550	cxl_driver_unregister(&cxl_pmem_region_driver);
551	cxl_driver_unregister(&cxl_nvdimm_driver);
552	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
553}
554
555MODULE_DESCRIPTION("CXL PMEM: Persistent Memory Support");
556MODULE_LICENSE("GPL v2");
557subsys_initcall(cxl_pmem_init);
558module_exit(cxl_pmem_exit);
559MODULE_IMPORT_NS("CXL");
560MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
561MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
562MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);