vmci_doorbell.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * VMware VMCI Driver
  4 *
  5 * Copyright (C) 2012 VMware, Inc. All rights reserved.
  6 */
  7
  8#include <linux/vmw_vmci_defs.h>
  9#include <linux/vmw_vmci_api.h>
 10#include <linux/completion.h>
 11#include <linux/hash.h>
 12#include <linux/kernel.h>
 13#include <linux/list.h>
 14#include <linux/module.h>
 15#include <linux/sched.h>
 16#include <linux/slab.h>
 17
 18#include "vmci_datagram.h"
 19#include "vmci_doorbell.h"
 20#include "vmci_resource.h"
 21#include "vmci_driver.h"
 22#include "vmci_route.h"
 23
 24
 25#define VMCI_DOORBELL_INDEX_BITS	6
 26#define VMCI_DOORBELL_INDEX_TABLE_SIZE	(1 << VMCI_DOORBELL_INDEX_BITS)
 27#define VMCI_DOORBELL_HASH(_idx)	hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)
 28
 29/*
 30 * DoorbellEntry describes the a doorbell notification handle allocated by the
 31 * host.
 32 */
 33struct dbell_entry {
 34	struct vmci_resource resource;
 35	struct hlist_node node;
 36	struct work_struct work;
 37	vmci_callback notify_cb;
 38	void *client_data;
 39	u32 idx;
 40	u32 priv_flags;
 41	bool run_delayed;
 42	atomic_t active;	/* Only used by guest personality */
 43};
 44
 45/* The VMCI index table keeps track of currently registered doorbells. */
 46struct dbell_index_table {
 47	spinlock_t lock;	/* Index table lock */
 48	struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
 49};
 50
 51static struct dbell_index_table vmci_doorbell_it = {
 52	.lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),
 53};
 54
 55/*
 56 * The max_notify_idx is one larger than the currently known bitmap index in
 57 * use, and is used to determine how much of the bitmap needs to be scanned.
 58 */
 59static u32 max_notify_idx;
 60
 61/*
 62 * The notify_idx_count is used for determining whether there are free entries
 63 * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
 64 */
 65static u32 notify_idx_count;
 66
 67/*
 68 * The last_notify_idx_reserved is used to track the last index handed out - in
 69 * the case where multiple handles share a notification index, we hand out
 70 * indexes round robin based on last_notify_idx_reserved.
 71 */
 72static u32 last_notify_idx_reserved;
 73
 74/* This is a one entry cache used to by the index allocation. */
 75static u32 last_notify_idx_released = PAGE_SIZE;
 76
 77
 78/*
 79 * Utility function that retrieves the privilege flags associated
 80 * with a given doorbell handle. For guest endpoints, the
 81 * privileges are determined by the context ID, but for host
 82 * endpoints privileges are associated with the complete
 83 * handle. Hypervisor endpoints are not yet supported.
 84 */
 85int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)
 86{
 87	if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
 88		return VMCI_ERROR_INVALID_ARGS;
 89
 90	if (handle.context == VMCI_HOST_CONTEXT_ID) {
 91		struct dbell_entry *entry;
 92		struct vmci_resource *resource;
 93
 94		resource = vmci_resource_by_handle(handle,
 95						   VMCI_RESOURCE_TYPE_DOORBELL);
 96		if (!resource)
 97			return VMCI_ERROR_NOT_FOUND;
 98
 99		entry = container_of(resource, struct dbell_entry, resource);
100		*priv_flags = entry->priv_flags;
101		vmci_resource_put(resource);
102	} else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
103		/*
104		 * Hypervisor endpoints for notifications are not
105		 * supported (yet).
106		 */
107		return VMCI_ERROR_INVALID_ARGS;
108	} else {
109		*priv_flags = vmci_context_get_priv_flags(handle.context);
110	}
111
112	return VMCI_SUCCESS;
113}
114
115/*
116 * Find doorbell entry by bitmap index.
117 */
118static struct dbell_entry *dbell_index_table_find(u32 idx)
119{
120	u32 bucket = VMCI_DOORBELL_HASH(idx);
121	struct dbell_entry *dbell;
122
123	hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],
124			     node) {
125		if (idx == dbell->idx)
126			return dbell;
127	}
128
129	return NULL;
130}
131
132/*
133 * Add the given entry to the index table.  This willi take a reference to the
134 * entry's resource so that the entry is not deleted before it is removed from
135 * the * table.
136 */
137static void dbell_index_table_add(struct dbell_entry *entry)
138{
139	u32 bucket;
140	u32 new_notify_idx;
141
142	vmci_resource_get(&entry->resource);
143
144	spin_lock_bh(&vmci_doorbell_it.lock);
145
146	/*
147	 * Below we try to allocate an index in the notification
148	 * bitmap with "not too much" sharing between resources. If we
149	 * use less that the full bitmap, we either add to the end if
150	 * there are no unused flags within the currently used area,
151	 * or we search for unused ones. If we use the full bitmap, we
152	 * allocate the index round robin.
153	 */
154	if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
155		if (last_notify_idx_released < max_notify_idx &&
156		    !dbell_index_table_find(last_notify_idx_released)) {
157			new_notify_idx = last_notify_idx_released;
158			last_notify_idx_released = PAGE_SIZE;
159		} else {
160			bool reused = false;
161			new_notify_idx = last_notify_idx_reserved;
162			if (notify_idx_count + 1 < max_notify_idx) {
163				do {
164					if (!dbell_index_table_find
165					    (new_notify_idx)) {
166						reused = true;
167						break;
168					}
169					new_notify_idx = (new_notify_idx + 1) %
170					    max_notify_idx;
171				} while (new_notify_idx !=
172					 last_notify_idx_released);
173			}
174			if (!reused) {
175				new_notify_idx = max_notify_idx;
176				max_notify_idx++;
177			}
178		}
179	} else {
180		new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
181	}
182
183	last_notify_idx_reserved = new_notify_idx;
184	notify_idx_count++;
185
186	entry->idx = new_notify_idx;
187	bucket = VMCI_DOORBELL_HASH(entry->idx);
188	hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);
189
190	spin_unlock_bh(&vmci_doorbell_it.lock);
191}
192
193/*
194 * Remove the given entry from the index table.  This will release() the
195 * entry's resource.
196 */
197static void dbell_index_table_remove(struct dbell_entry *entry)
198{
199	spin_lock_bh(&vmci_doorbell_it.lock);
200
201	hlist_del_init(&entry->node);
202
203	notify_idx_count--;
204	if (entry->idx == max_notify_idx - 1) {
205		/*
206		 * If we delete an entry with the maximum known
207		 * notification index, we take the opportunity to
208		 * prune the current max. As there might be other
209		 * unused indices immediately below, we lower the
210		 * maximum until we hit an index in use.
211		 */
212		while (max_notify_idx > 0 &&
213		       !dbell_index_table_find(max_notify_idx - 1))
214			max_notify_idx--;
215	}
216
217	last_notify_idx_released = entry->idx;
218
219	spin_unlock_bh(&vmci_doorbell_it.lock);
220
221	vmci_resource_put(&entry->resource);
222}
223
224/*
225 * Creates a link between the given doorbell handle and the given
226 * index in the bitmap in the device backend. A notification state
227 * is created in hypervisor.
228 */
229static int dbell_link(struct vmci_handle handle, u32 notify_idx)
230{
231	struct vmci_doorbell_link_msg link_msg;
232
233	link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
234					    VMCI_DOORBELL_LINK);
235	link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
236	link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
237	link_msg.handle = handle;
238	link_msg.notify_idx = notify_idx;
239
240	return vmci_send_datagram(&link_msg.hdr);
241}
242
243/*
244 * Unlinks the given doorbell handle from an index in the bitmap in
245 * the device backend. The notification state is destroyed in hypervisor.
246 */
247static int dbell_unlink(struct vmci_handle handle)
248{
249	struct vmci_doorbell_unlink_msg unlink_msg;
250
251	unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
252					      VMCI_DOORBELL_UNLINK);
253	unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
254	unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
255	unlink_msg.handle = handle;
256
257	return vmci_send_datagram(&unlink_msg.hdr);
258}
259
260/*
261 * Calls the specified callback in a delayed context.
262 */
263static void dbell_delayed_dispatch(struct work_struct *work)
264{
265	struct dbell_entry *entry = container_of(work,
266						 struct dbell_entry, work);
267
268	entry->notify_cb(entry->client_data);
269	vmci_resource_put(&entry->resource);
270}
271
272/*
273 * Dispatches a doorbell notification to the host context.
274 */
275int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)
276{
277	struct dbell_entry *entry;
278	struct vmci_resource *resource;
279
280	if (vmci_handle_is_invalid(handle)) {
281		pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",
282			 handle.context, handle.resource);
283		return VMCI_ERROR_INVALID_ARGS;
284	}
285
286	resource = vmci_resource_by_handle(handle,
287					   VMCI_RESOURCE_TYPE_DOORBELL);
288	if (!resource) {
289		pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",
290			 handle.context, handle.resource);
291		return VMCI_ERROR_NOT_FOUND;
292	}
293
294	entry = container_of(resource, struct dbell_entry, resource);
295	if (entry->run_delayed) {
296		if (!schedule_work(&entry->work))
297			vmci_resource_put(resource);
298	} else {
299		entry->notify_cb(entry->client_data);
300		vmci_resource_put(resource);
301	}
302
303	return VMCI_SUCCESS;
304}
305
306/*
307 * Register the notification bitmap with the host.
308 */
309bool vmci_dbell_register_notification_bitmap(u64 bitmap_ppn)
310{
311	int result;
312	struct vmci_notify_bm_set_msg bitmap_set_msg = { };
313
314	bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
315						  VMCI_SET_NOTIFY_BITMAP);
316	bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
317	bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -
318	    VMCI_DG_HEADERSIZE;
319	if (vmci_use_ppn64())
320		bitmap_set_msg.bitmap_ppn64 = bitmap_ppn;
321	else
322		bitmap_set_msg.bitmap_ppn32 = (u32) bitmap_ppn;
323
324	result = vmci_send_datagram(&bitmap_set_msg.hdr);
325	if (result != VMCI_SUCCESS) {
326		pr_devel("Failed to register (PPN=%llu) as notification bitmap (error=%d)\n",
327			 bitmap_ppn, result);
328		return false;
329	}
330	return true;
331}
332
333/*
334 * Executes or schedules the handlers for a given notify index.
335 */
336static void dbell_fire_entries(u32 notify_idx)
337{
338	u32 bucket = VMCI_DOORBELL_HASH(notify_idx);
339	struct dbell_entry *dbell;
340
341	spin_lock_bh(&vmci_doorbell_it.lock);
342
343	hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {
344		if (dbell->idx == notify_idx &&
345		    atomic_read(&dbell->active) == 1) {
346			if (dbell->run_delayed) {
347				vmci_resource_get(&dbell->resource);
348				if (!schedule_work(&dbell->work))
349					vmci_resource_put(&dbell->resource);
350			} else {
351				dbell->notify_cb(dbell->client_data);
352			}
353		}
354	}
355
356	spin_unlock_bh(&vmci_doorbell_it.lock);
357}
358
359/*
360 * Scans the notification bitmap, collects pending notifications,
361 * resets the bitmap and invokes appropriate callbacks.
362 */
363void vmci_dbell_scan_notification_entries(u8 *bitmap)
364{
365	u32 idx;
366
367	for (idx = 0; idx < max_notify_idx; idx++) {
368		if (bitmap[idx] & 0x1) {
369			bitmap[idx] &= ~1;
370			dbell_fire_entries(idx);
371		}
372	}
373}
374
375/*
376 * vmci_doorbell_create() - Creates a doorbell
377 * @handle:     A handle used to track the resource.  Can be invalid.
378 * @flags:      Flag that determines context of callback.
379 * @priv_flags: Privileges flags.
380 * @notify_cb:  The callback to be ivoked when the doorbell fires.
381 * @client_data:        A parameter to be passed to the callback.
382 *
383 * Creates a doorbell with the given callback. If the handle is
384 * VMCI_INVALID_HANDLE, a free handle will be assigned, if
385 * possible. The callback can be run immediately (potentially with
386 * locks held - the default) or delayed (in a kernel thread) by
387 * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution
388 * is selected, a given callback may not be run if the kernel is
389 * unable to allocate memory for the delayed execution (highly
390 * unlikely).
391 */
392int vmci_doorbell_create(struct vmci_handle *handle,
393			 u32 flags,
394			 u32 priv_flags,
395			 vmci_callback notify_cb, void *client_data)
396{
397	struct dbell_entry *entry;
398	struct vmci_handle new_handle;
399	int result;
400
401	if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
402	    priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
403		return VMCI_ERROR_INVALID_ARGS;
404
405	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
406	if (entry == NULL) {
407		pr_warn("Failed allocating memory for datagram entry\n");
408		return VMCI_ERROR_NO_MEM;
409	}
410
411	if (vmci_handle_is_invalid(*handle)) {
412		u32 context_id = vmci_get_context_id();
413
414		if (context_id == VMCI_INVALID_ID) {
415			pr_warn("Failed to get context ID\n");
416			result = VMCI_ERROR_NO_RESOURCES;
417			goto free_mem;
418		}
419
420		/* Let resource code allocate a free ID for us */
421		new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
422	} else {
423		bool valid_context = false;
424
425		/*
426		 * Validate the handle.  We must do both of the checks below
427		 * because we can be acting as both a host and a guest at the
428		 * same time. We always allow the host context ID, since the
429		 * host functionality is in practice always there with the
430		 * unified driver.
431		 */
432		if (handle->context == VMCI_HOST_CONTEXT_ID ||
433		    (vmci_guest_code_active() &&
434		     vmci_get_context_id() == handle->context)) {
435			valid_context = true;
436		}
437
438		if (!valid_context || handle->resource == VMCI_INVALID_ID) {
439			pr_devel("Invalid argument (handle=0x%x:0x%x)\n",
440				 handle->context, handle->resource);
441			result = VMCI_ERROR_INVALID_ARGS;
442			goto free_mem;
443		}
444
445		new_handle = *handle;
446	}
447
448	entry->idx = 0;
449	INIT_HLIST_NODE(&entry->node);
450	entry->priv_flags = priv_flags;
451	INIT_WORK(&entry->work, dbell_delayed_dispatch);
452	entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;
453	entry->notify_cb = notify_cb;
454	entry->client_data = client_data;
455	atomic_set(&entry->active, 0);
456
457	result = vmci_resource_add(&entry->resource,
458				   VMCI_RESOURCE_TYPE_DOORBELL,
459				   new_handle);
460	if (result != VMCI_SUCCESS) {
461		pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",
462			new_handle.context, new_handle.resource, result);
463		goto free_mem;
464	}
465
466	new_handle = vmci_resource_handle(&entry->resource);
467	if (vmci_guest_code_active()) {
468		dbell_index_table_add(entry);
469		result = dbell_link(new_handle, entry->idx);
470		if (VMCI_SUCCESS != result)
471			goto destroy_resource;
472
473		atomic_set(&entry->active, 1);
474	}
475
476	*handle = new_handle;
477
478	return result;
479
480 destroy_resource:
481	dbell_index_table_remove(entry);
482	vmci_resource_remove(&entry->resource);
483 free_mem:
484	kfree(entry);
485	return result;
486}
487EXPORT_SYMBOL_GPL(vmci_doorbell_create);
488
489/*
490 * vmci_doorbell_destroy() - Destroy a doorbell.
491 * @handle:     The handle tracking the resource.
492 *
493 * Destroys a doorbell previously created with vmcii_doorbell_create. This
494 * operation may block waiting for a callback to finish.
495 */
496int vmci_doorbell_destroy(struct vmci_handle handle)
497{
498	struct dbell_entry *entry;
499	struct vmci_resource *resource;
500
501	if (vmci_handle_is_invalid(handle))
502		return VMCI_ERROR_INVALID_ARGS;
503
504	resource = vmci_resource_by_handle(handle,
505					   VMCI_RESOURCE_TYPE_DOORBELL);
506	if (!resource) {
507		pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",
508			 handle.context, handle.resource);
509		return VMCI_ERROR_NOT_FOUND;
510	}
511
512	entry = container_of(resource, struct dbell_entry, resource);
513
514	if (!hlist_unhashed(&entry->node)) {
515		int result;
516
517		dbell_index_table_remove(entry);
518
519		result = dbell_unlink(handle);
520		if (VMCI_SUCCESS != result) {
521
522			/*
523			 * The only reason this should fail would be
524			 * an inconsistency between guest and
525			 * hypervisor state, where the guest believes
526			 * it has an active registration whereas the
527			 * hypervisor doesn't. One case where this may
528			 * happen is if a doorbell is unregistered
529			 * following a hibernation at a time where the
530			 * doorbell state hasn't been restored on the
531			 * hypervisor side yet. Since the handle has
532			 * now been removed in the guest, we just
533			 * print a warning and return success.
534			 */
535			pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",
536				 handle.context, handle.resource, result);
537		}
538	}
539
540	/*
541	 * Now remove the resource from the table.  It might still be in use
542	 * after this, in a callback or still on the delayed work queue.
543	 */
544	vmci_resource_put(&entry->resource);
545	vmci_resource_remove(&entry->resource);
546
547	kfree(entry);
548
549	return VMCI_SUCCESS;
550}
551EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);