habanalabs_drv.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Copyright 2016-2021 HabanaLabs, Ltd.
  5 * All Rights Reserved.
  6 *
  7 */
  8
  9#define pr_fmt(fmt)		"habanalabs: " fmt
 10
 11#include "habanalabs.h"
 12#include "../include/hw_ip/pci/pci_general.h"
 13
 14#include <linux/pci.h>
 15#include <linux/module.h>
 16#include <linux/vmalloc.h>
 17#include <linux/version.h>
 18
 19#include <drm/drm_accel.h>
 20#include <drm/drm_drv.h>
 21#include <drm/drm_ioctl.h>
 22
 23#define CREATE_TRACE_POINTS
 24#include <trace/events/habanalabs.h>
 25
 26#define HL_DRIVER_AUTHOR	"HabanaLabs Kernel Driver Team"
 27
 28#define HL_DRIVER_DESC		"Driver for HabanaLabs's AI Accelerators"
 29
 30MODULE_AUTHOR(HL_DRIVER_AUTHOR);
 31MODULE_DESCRIPTION(HL_DRIVER_DESC);
 32MODULE_LICENSE("GPL v2");
 33
 34static int hl_major;
 35static DEFINE_IDR(hl_devs_idr);
 36static DEFINE_MUTEX(hl_devs_idr_lock);
 37
 38#define HL_DEFAULT_TIMEOUT_LOCKED	30	/* 30 seconds */
 39#define GAUDI_DEFAULT_TIMEOUT_LOCKED	600	/* 10 minutes */
 40
 41static int timeout_locked = HL_DEFAULT_TIMEOUT_LOCKED;
 42static int reset_on_lockup = 1;
 43static int memory_scrub;
 44static ulong boot_error_status_mask = ULONG_MAX;
 45
 46module_param(timeout_locked, int, 0444);
 47MODULE_PARM_DESC(timeout_locked,
 48	"Device lockup timeout in seconds (0 = disabled, default 30s)");
 49
 50module_param(reset_on_lockup, int, 0444);
 51MODULE_PARM_DESC(reset_on_lockup,
 52	"Do device reset on lockup (0 = no, 1 = yes, default yes)");
 53
 54module_param(memory_scrub, int, 0444);
 55MODULE_PARM_DESC(memory_scrub,
 56	"Scrub device memory in various states (0 = no, 1 = yes, default no)");
 57
 58module_param(boot_error_status_mask, ulong, 0444);
 59MODULE_PARM_DESC(boot_error_status_mask,
 60	"Mask of the error status during device CPU boot (If bitX is cleared then error X is masked. Default all 1's)");
 61
 62#define PCI_IDS_GOYA			0x0001
 63#define PCI_IDS_GAUDI			0x1000
 64#define PCI_IDS_GAUDI_SEC		0x1010
 65
 66#define PCI_IDS_GAUDI2			0x1020
 67
 68static const struct pci_device_id ids[] = {
 69	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GOYA), },
 70	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI), },
 71	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI_SEC), },
 72	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI2), },
 73	{ 0, }
 74};
 75MODULE_DEVICE_TABLE(pci, ids);
 76
 77static const struct drm_ioctl_desc hl_drm_ioctls[] = {
 78	DRM_IOCTL_DEF_DRV(HL_INFO, hl_info_ioctl, 0),
 79	DRM_IOCTL_DEF_DRV(HL_CB, hl_cb_ioctl, 0),
 80	DRM_IOCTL_DEF_DRV(HL_CS, hl_cs_ioctl, 0),
 81	DRM_IOCTL_DEF_DRV(HL_WAIT_CS, hl_wait_ioctl, 0),
 82	DRM_IOCTL_DEF_DRV(HL_MEMORY, hl_mem_ioctl, 0),
 83	DRM_IOCTL_DEF_DRV(HL_DEBUG, hl_debug_ioctl, 0),
 84};
 85
 86static const struct file_operations hl_fops = {
 87	.owner = THIS_MODULE,
 88	.open = accel_open,
 89	.release = drm_release,
 90	.unlocked_ioctl = drm_ioctl,
 91	.compat_ioctl = drm_compat_ioctl,
 92	.llseek = noop_llseek,
 93	.mmap = hl_mmap
 94};
 95
 96static const struct drm_driver hl_driver = {
 97	.driver_features = DRIVER_COMPUTE_ACCEL,
 98
 99	.name = HL_NAME,
100	.desc = HL_DRIVER_DESC,
101	.major = LINUX_VERSION_MAJOR,
102	.minor = LINUX_VERSION_PATCHLEVEL,
103	.patchlevel = LINUX_VERSION_SUBLEVEL,
104
105	.fops = &hl_fops,
106	.open = hl_device_open,
107	.postclose = hl_device_release,
108	.ioctls = hl_drm_ioctls,
109	.num_ioctls = ARRAY_SIZE(hl_drm_ioctls)
110};
111
112/*
113 * get_asic_type - translate device id to asic type
114 *
115 * @hdev: pointer to habanalabs device structure.
116 *
117 * Translate device id and revision id to asic type.
118 * In case of unidentified device, return -1
119 */
120static enum hl_asic_type get_asic_type(struct hl_device *hdev)
121{
122	struct pci_dev *pdev = hdev->pdev;
123	enum hl_asic_type asic_type = ASIC_INVALID;
124
125	switch (pdev->device) {
126	case PCI_IDS_GOYA:
127		asic_type = ASIC_GOYA;
128		break;
129	case PCI_IDS_GAUDI:
130		asic_type = ASIC_GAUDI;
131		break;
132	case PCI_IDS_GAUDI_SEC:
133		asic_type = ASIC_GAUDI_SEC;
134		break;
135	case PCI_IDS_GAUDI2:
136		switch (pdev->revision) {
137		case REV_ID_A:
138			asic_type = ASIC_GAUDI2;
139			break;
140		case REV_ID_B:
141			asic_type = ASIC_GAUDI2B;
142			break;
143		case REV_ID_C:
144			asic_type = ASIC_GAUDI2C;
145			break;
146		case REV_ID_D:
147			asic_type = ASIC_GAUDI2D;
148			break;
149		default:
150			break;
151		}
152		break;
153	default:
154		break;
155	}
156
157	return asic_type;
158}
159
160static bool is_asic_secured(enum hl_asic_type asic_type)
161{
162	switch (asic_type) {
163	case ASIC_GAUDI_SEC:
164		return true;
165	default:
166		return false;
167	}
168}
169
170/*
171 * hl_device_open() - open function for habanalabs device.
172 * @ddev: pointer to DRM device structure.
173 * @file: pointer to DRM file private data structure.
174 *
175 * Called when process opens an habanalabs device.
176 */
177int hl_device_open(struct drm_device *ddev, struct drm_file *file_priv)
178{
179	struct hl_device *hdev = to_hl_device(ddev);
180	enum hl_device_status status;
181	struct hl_fpriv *hpriv;
182	int rc;
183
184	hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
185	if (!hpriv)
186		return -ENOMEM;
187
188	hpriv->hdev = hdev;
189	mutex_init(&hpriv->notifier_event.lock);
190	mutex_init(&hpriv->restore_phase_mutex);
191	mutex_init(&hpriv->ctx_lock);
192	kref_init(&hpriv->refcount);
193
194	hl_ctx_mgr_init(&hpriv->ctx_mgr);
195	hl_mem_mgr_init(hpriv->hdev->dev, &hpriv->mem_mgr);
196
197	hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);
198
199	mutex_lock(&hdev->fpriv_list_lock);
200
201	if (!hl_device_operational(hdev, &status)) {
202		dev_dbg_ratelimited(hdev->dev,
203			"Can't open %s because it is %s\n",
204			dev_name(hdev->dev), hdev->status[status]);
205
206		if (status == HL_DEVICE_STATUS_IN_RESET ||
207					status == HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE)
208			rc = -EAGAIN;
209		else
210			rc = -EPERM;
211
212		goto out_err;
213	}
214
215	if (hdev->is_in_dram_scrub) {
216		dev_dbg_ratelimited(hdev->dev,
217			"Can't open %s during dram scrub\n",
218			dev_name(hdev->dev));
219		rc = -EAGAIN;
220		goto out_err;
221	}
222
223	if (hdev->compute_ctx_in_release) {
224		dev_dbg_ratelimited(hdev->dev,
225			"Can't open %s because another user is still releasing it\n",
226			dev_name(hdev->dev));
227		rc = -EAGAIN;
228		goto out_err;
229	}
230
231	if (hdev->is_compute_ctx_active) {
232		dev_dbg_ratelimited(hdev->dev,
233			"Can't open %s because another user is working on it\n",
234			dev_name(hdev->dev));
235		rc = -EBUSY;
236		goto out_err;
237	}
238
239	rc = hl_ctx_create(hdev, hpriv);
240	if (rc) {
241		dev_err(hdev->dev, "Failed to create context %d\n", rc);
242		goto out_err;
243	}
244
245	list_add(&hpriv->dev_node, &hdev->fpriv_list);
246	mutex_unlock(&hdev->fpriv_list_lock);
247
248	hdev->asic_funcs->send_device_activity(hdev, true);
249
250	hl_debugfs_add_file(hpriv);
251
252	hl_enable_err_info_capture(&hdev->captured_err_info);
253
254	hdev->open_counter++;
255	hdev->last_successful_open_jif = jiffies;
256	hdev->last_successful_open_ktime = ktime_get();
257
258	file_priv->driver_priv = hpriv;
259	hpriv->file_priv = file_priv;
260
261	return 0;
262
263out_err:
264	mutex_unlock(&hdev->fpriv_list_lock);
265	hl_mem_mgr_fini(&hpriv->mem_mgr, NULL);
266	hl_mem_mgr_idr_destroy(&hpriv->mem_mgr);
267	hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
268	mutex_destroy(&hpriv->ctx_lock);
269	mutex_destroy(&hpriv->restore_phase_mutex);
270	mutex_destroy(&hpriv->notifier_event.lock);
271	put_pid(hpriv->taskpid);
272
273	kfree(hpriv);
274
275	return rc;
276}
277
278int hl_device_open_ctrl(struct inode *inode, struct file *filp)
279{
280	struct hl_device *hdev;
281	struct hl_fpriv *hpriv;
282	int rc;
283
284	mutex_lock(&hl_devs_idr_lock);
285	hdev = idr_find(&hl_devs_idr, iminor(inode));
286	mutex_unlock(&hl_devs_idr_lock);
287
288	if (!hdev) {
289		pr_err("Couldn't find device %d:%d\n",
290			imajor(inode), iminor(inode));
291		return -ENXIO;
292	}
293
294	hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
295	if (!hpriv)
296		return -ENOMEM;
297
298	/* Prevent other routines from reading partial hpriv data by
299	 * initializing hpriv fields before inserting it to the list
300	 */
301	hpriv->hdev = hdev;
302	filp->private_data = hpriv;
303
304	nonseekable_open(inode, filp);
305
306	hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);
307
308	mutex_lock(&hdev->fpriv_ctrl_list_lock);
309
310	if (!hl_ctrl_device_operational(hdev, NULL)) {
311		dev_dbg_ratelimited(hdev->dev_ctrl,
312			"Can't open %s because it is disabled\n",
313			dev_name(hdev->dev_ctrl));
314		rc = -EPERM;
315		goto out_err;
316	}
317
318	list_add(&hpriv->dev_node, &hdev->fpriv_ctrl_list);
319	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
320
321	return 0;
322
323out_err:
324	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
325	filp->private_data = NULL;
326	put_pid(hpriv->taskpid);
327
328	kfree(hpriv);
329
330	return rc;
331}
332
333static void set_driver_behavior_per_device(struct hl_device *hdev)
334{
335	hdev->nic_ports_mask = 0;
336	hdev->fw_components = FW_TYPE_ALL_TYPES;
337	hdev->cpu_queues_enable = 1;
338	hdev->pldm = 0;
339	hdev->hard_reset_on_fw_events = 1;
340	hdev->bmc_enable = 1;
341	hdev->reset_on_preboot_fail = 1;
342	hdev->heartbeat = 1;
343}
344
345static void copy_kernel_module_params_to_device(struct hl_device *hdev)
346{
347	hdev->asic_prop.fw_security_enabled = is_asic_secured(hdev->asic_type);
348
349	hdev->major = hl_major;
350	hdev->memory_scrub = memory_scrub;
351	hdev->reset_on_lockup = reset_on_lockup;
352	hdev->boot_error_status_mask = boot_error_status_mask;
353}
354
355static void fixup_device_params_per_asic(struct hl_device *hdev, int timeout)
356{
357	switch (hdev->asic_type) {
358	case ASIC_GAUDI:
359	case ASIC_GAUDI_SEC:
360		/* If user didn't request a different timeout than the default one, we have
361		 * a different default timeout for Gaudi
362		 */
363		if (timeout == HL_DEFAULT_TIMEOUT_LOCKED)
364			hdev->timeout_jiffies = secs_to_jiffies(GAUDI_DEFAULT_TIMEOUT_LOCKED);
365
366		hdev->reset_upon_device_release = 0;
367		break;
368
369	case ASIC_GOYA:
370		hdev->reset_upon_device_release = 0;
371		break;
372
373	default:
374		hdev->reset_upon_device_release = 1;
375		break;
376	}
377}
378
379static int fixup_device_params(struct hl_device *hdev)
380{
381	int tmp_timeout;
382
383	tmp_timeout = timeout_locked;
384
385	hdev->fw_poll_interval_usec = HL_FW_STATUS_POLL_INTERVAL_USEC;
386	hdev->fw_comms_poll_interval_usec = HL_FW_STATUS_POLL_INTERVAL_USEC;
387
388	if (tmp_timeout)
389		hdev->timeout_jiffies = secs_to_jiffies(tmp_timeout);
390	else
391		hdev->timeout_jiffies = MAX_SCHEDULE_TIMEOUT;
392
393	hdev->stop_on_err = true;
394	hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
395	hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
396
397	/* Enable only after the initialization of the device */
398	hdev->disabled = true;
399
400	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) &&
401			(hdev->fw_components & ~FW_TYPE_PREBOOT_CPU)) {
402		pr_err("Preboot must be set along with other components");
403		return -EINVAL;
404	}
405
406	/* If CPU queues not enabled, no way to do heartbeat */
407	if (!hdev->cpu_queues_enable)
408		hdev->heartbeat = 0;
409	fixup_device_params_per_asic(hdev, tmp_timeout);
410
411	return 0;
412}
413
414static int allocate_device_id(struct hl_device *hdev)
415{
416	int id;
417
418	mutex_lock(&hl_devs_idr_lock);
419	id = idr_alloc(&hl_devs_idr, hdev, 0, HL_MAX_MINORS, GFP_KERNEL);
420	mutex_unlock(&hl_devs_idr_lock);
421
422	if (id < 0) {
423		if (id == -ENOSPC)
424			pr_err("too many devices in the system\n");
425		return -EBUSY;
426	}
427
428	hdev->id = id;
429
430	/*
431	 * Firstly initialized with the internal device ID.
432	 * Will be updated later after the DRM device registration to hold the minor ID.
433	 */
434	hdev->cdev_idx = hdev->id;
435
436	return 0;
437}
438
439/**
440 * create_hdev - create habanalabs device instance
441 *
442 * @dev: will hold the pointer to the new habanalabs device structure
443 * @pdev: pointer to the pci device
444 *
445 * Allocate memory for habanalabs device and initialize basic fields
446 * Identify the ASIC type
447 * Allocate ID (minor) for the device (only for real devices)
448 */
449static int create_hdev(struct hl_device **dev, struct pci_dev *pdev)
450{
451	struct hl_device *hdev;
452	int rc;
453
454	*dev = NULL;
455
456	hdev = devm_drm_dev_alloc(&pdev->dev, &hl_driver, struct hl_device, drm);
457	if (IS_ERR(hdev))
458		return PTR_ERR(hdev);
459
460	hdev->dev = hdev->drm.dev;
461
462	/* Will be NULL in case of simulator device */
463	hdev->pdev = pdev;
464
465	/* Assign status description string */
466	strscpy(hdev->status[HL_DEVICE_STATUS_OPERATIONAL], "operational", HL_STR_MAX);
467	strscpy(hdev->status[HL_DEVICE_STATUS_IN_RESET], "in reset", HL_STR_MAX);
468	strscpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION], "disabled", HL_STR_MAX);
469	strscpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET], "needs reset", HL_STR_MAX);
470	strscpy(hdev->status[HL_DEVICE_STATUS_IN_DEVICE_CREATION],
471				"in device creation", HL_STR_MAX);
472	strscpy(hdev->status[HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE],
473				"in reset after device release", HL_STR_MAX);
474
475
476	/* First, we must find out which ASIC are we handling. This is needed
477	 * to configure the behavior of the driver (kernel parameters)
478	 */
479	hdev->asic_type = get_asic_type(hdev);
480	if (hdev->asic_type == ASIC_INVALID) {
481		dev_err(&pdev->dev, "Unsupported ASIC\n");
482		rc = -ENODEV;
483		goto out_err;
484	}
485
486	copy_kernel_module_params_to_device(hdev);
487
488	set_driver_behavior_per_device(hdev);
489
490	fixup_device_params(hdev);
491
492	rc = allocate_device_id(hdev);
493	if (rc)
494		goto out_err;
495
496	*dev = hdev;
497
498	return 0;
499
500out_err:
501	return rc;
502}
503
504/*
505 * destroy_hdev - destroy habanalabs device instance
506 *
507 * @dev: pointer to the habanalabs device structure
508 *
509 */
510static void destroy_hdev(struct hl_device *hdev)
511{
512	/* Remove device from the device list */
513	mutex_lock(&hl_devs_idr_lock);
514	idr_remove(&hl_devs_idr, hdev->id);
515	mutex_unlock(&hl_devs_idr_lock);
516
517}
518
519static int hl_pmops_suspend(struct device *dev)
520{
521	struct hl_device *hdev = dev_get_drvdata(dev);
522
523	pr_debug("Going to suspend PCI device\n");
524
525	if (!hdev) {
526		pr_err("device pointer is NULL in suspend\n");
527		return 0;
528	}
529
530	return hl_device_suspend(hdev);
531}
532
533static int hl_pmops_resume(struct device *dev)
534{
535	struct hl_device *hdev = dev_get_drvdata(dev);
536
537	pr_debug("Going to resume PCI device\n");
538
539	if (!hdev) {
540		pr_err("device pointer is NULL in resume\n");
541		return 0;
542	}
543
544	return hl_device_resume(hdev);
545}
546
547/**
548 * hl_pci_probe - probe PCI habanalabs devices
549 *
550 * @pdev: pointer to pci device
551 * @id: pointer to pci device id structure
552 *
553 * Standard PCI probe function for habanalabs device.
554 * Create a new habanalabs device and initialize it according to the
555 * device's type
556 */
557static int hl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
558{
559	struct hl_device *hdev;
560	int rc;
561
562	dev_info(&pdev->dev, HL_NAME
563		 " device found [%04x:%04x] (rev %x)\n",
564		 (int)pdev->vendor, (int)pdev->device, (int)pdev->revision);
565
566	rc = create_hdev(&hdev, pdev);
567	if (rc)
568		return rc;
569
570	pci_set_drvdata(pdev, hdev);
571
572	rc = hl_device_init(hdev);
573	if (rc) {
574		dev_err(&pdev->dev, "Fatal error during habanalabs device init\n");
575		rc = -ENODEV;
576		goto disable_device;
577	}
578
579	return 0;
580
581disable_device:
582	pci_set_drvdata(pdev, NULL);
583	destroy_hdev(hdev);
584
585	return rc;
586}
587
588/*
589 * hl_pci_remove - remove PCI habanalabs devices
590 *
591 * @pdev: pointer to pci device
592 *
593 * Standard PCI remove function for habanalabs device
594 */
595static void hl_pci_remove(struct pci_dev *pdev)
596{
597	struct hl_device *hdev;
598
599	hdev = pci_get_drvdata(pdev);
600	if (!hdev)
601		return;
602
603	hl_device_fini(hdev);
604	pci_set_drvdata(pdev, NULL);
605	destroy_hdev(hdev);
606}
607
608/**
609 * hl_pci_err_detected - a PCI bus error detected on this device
610 *
611 * @pdev: pointer to pci device
612 * @state: PCI error type
613 *
614 * Called by the PCI subsystem whenever a non-correctable
615 * PCI bus error is detected
616 */
617static pci_ers_result_t
618hl_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state)
619{
620	struct hl_device *hdev = pci_get_drvdata(pdev);
621	enum pci_ers_result result;
622
623	switch (state) {
624	case pci_channel_io_normal:
625		dev_warn(hdev->dev, "PCI normal state error detected\n");
626		return PCI_ERS_RESULT_CAN_RECOVER;
627
628	case pci_channel_io_frozen:
629		dev_warn(hdev->dev, "PCI frozen state error detected\n");
630		result = PCI_ERS_RESULT_NEED_RESET;
631		break;
632
633	case pci_channel_io_perm_failure:
634		dev_warn(hdev->dev, "PCI failure state error detected\n");
635		result = PCI_ERS_RESULT_DISCONNECT;
636		break;
637
638	default:
639		result = PCI_ERS_RESULT_NONE;
640	}
641
642	hdev->asic_funcs->halt_engines(hdev, true, false);
643
644	return result;
645}
646
647/**
648 * hl_pci_err_resume - resume after a PCI slot reset
649 *
650 * @pdev: pointer to pci device
651 *
652 */
653static void hl_pci_err_resume(struct pci_dev *pdev)
654{
655	struct hl_device *hdev = pci_get_drvdata(pdev);
656
657	dev_warn(hdev->dev, "Resuming device after PCI slot reset\n");
658	hl_device_resume(hdev);
659}
660
661/**
662 * hl_pci_err_slot_reset - a PCI slot reset has just happened
663 *
664 * @pdev: pointer to pci device
665 *
666 * Determine if the driver can recover from the PCI slot reset
667 */
668static pci_ers_result_t hl_pci_err_slot_reset(struct pci_dev *pdev)
669{
670	struct hl_device *hdev = pci_get_drvdata(pdev);
671
672	dev_warn(hdev->dev, "PCI slot reset detected\n");
673
674	return PCI_ERS_RESULT_RECOVERED;
675}
676
677static void hl_pci_reset_prepare(struct pci_dev *pdev)
678{
679	struct hl_device *hdev;
680
681	hdev = pci_get_drvdata(pdev);
682	if (!hdev)
683		return;
684
685	hdev->disabled = true;
686}
687
688static void hl_pci_reset_done(struct pci_dev *pdev)
689{
690	struct hl_device *hdev;
691	u32 flags;
692
693	hdev = pci_get_drvdata(pdev);
694	if (!hdev)
695		return;
696
697	/*
698	 * Schedule a thread to trigger hard reset.
699	 * The reason for this handler, is for rare cases where the driver is up
700	 * and FLR occurs. This is valid only when working with no VM, so FW handles FLR
701	 * and resets the device. FW will go back preboot stage, so driver needs to perform
702	 * hard reset in order to load FW fit again.
703	 */
704	flags = HL_DRV_RESET_HARD | HL_DRV_RESET_BYPASS_REQ_TO_FW;
705
706	hl_device_reset(hdev, flags);
707}
708
709static const struct dev_pm_ops hl_pm_ops = {
710	.suspend = hl_pmops_suspend,
711	.resume = hl_pmops_resume,
712};
713
714static const struct pci_error_handlers hl_pci_err_handler = {
715	.error_detected = hl_pci_err_detected,
716	.slot_reset = hl_pci_err_slot_reset,
717	.resume = hl_pci_err_resume,
718	.reset_prepare = hl_pci_reset_prepare,
719	.reset_done = hl_pci_reset_done,
720};
721
722static struct pci_driver hl_pci_driver = {
723	.name = HL_NAME,
724	.id_table = ids,
725	.probe = hl_pci_probe,
726	.remove = hl_pci_remove,
727	.shutdown = hl_pci_remove,
728	.driver = {
729		.name = HL_NAME,
730		.pm = &hl_pm_ops,
731		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
732	},
733	.err_handler = &hl_pci_err_handler,
734};
735
736/*
737 * hl_init - Initialize the habanalabs kernel driver
738 */
739static int __init hl_init(void)
740{
741	int rc;
742	dev_t dev;
743
744	pr_info("loading driver\n");
745
746	rc = alloc_chrdev_region(&dev, 0, HL_MAX_MINORS, HL_NAME);
747	if (rc < 0) {
748		pr_err("unable to get major\n");
749		return rc;
750	}
751
752	hl_major = MAJOR(dev);
753
754	rc = pci_register_driver(&hl_pci_driver);
755	if (rc) {
756		pr_err("failed to register pci device\n");
757		goto remove_major;
758	}
759
760	pr_debug("driver loaded\n");
761
762	return 0;
763
764remove_major:
765	unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
766	return rc;
767}
768
769/*
770 * hl_exit - Release all resources of the habanalabs kernel driver
771 */
772static void __exit hl_exit(void)
773{
774	pci_unregister_driver(&hl_pci_driver);
775
776	unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
777
778	idr_destroy(&hl_devs_idr);
779
780	pr_debug("driver removed\n");
781}
782
783module_init(hl_init);
784module_exit(hl_exit);