folio_queue.h

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2/* Queue of folios definitions
  3 *
  4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
  5 * Written by David Howells (dhowells@redhat.com)
  6 *
  7 * See:
  8 *
  9 *	Documentation/core-api/folio_queue.rst
 10 *
 11 * for a description of the API.
 12 */
 13
 14#ifndef _LINUX_FOLIO_QUEUE_H
 15#define _LINUX_FOLIO_QUEUE_H
 16
 17#include <linux/pagevec.h>
 18#include <linux/mm.h>
 19
 20/*
 21 * Segment in a queue of running buffers.  Each segment can hold a number of
 22 * folios and a portion of the queue can be referenced with the ITER_FOLIOQ
 23 * iterator.  The possibility exists of inserting non-folio elements into the
 24 * queue (such as gaps).
 25 *
 26 * Explicit prev and next pointers are used instead of a list_head to make it
 27 * easier to add segments to tail and remove them from the head without the
 28 * need for a lock.
 29 */
 30struct folio_queue {
 31	struct folio_batch	vec;		/* Folios in the queue segment */
 32	u8			orders[PAGEVEC_SIZE]; /* Order of each folio */
 33	struct folio_queue	*next;		/* Next queue segment or NULL */
 34	struct folio_queue	*prev;		/* Previous queue segment of NULL */
 35	unsigned long		marks;		/* 1-bit mark per folio */
 36	unsigned long		marks2;		/* Second 1-bit mark per folio */
 37#if PAGEVEC_SIZE > BITS_PER_LONG
 38#error marks is not big enough
 39#endif
 40	unsigned int		rreq_id;
 41	unsigned int		debug_id;
 42};
 43
 44/**
 45 * folioq_init - Initialise a folio queue segment
 46 * @folioq: The segment to initialise
 47 * @rreq_id: The request identifier to use in tracelines.
 48 *
 49 * Initialise a folio queue segment and set an identifier to be used in traces.
 50 *
 51 * Note that the folio pointers are left uninitialised.
 52 */
 53static inline void folioq_init(struct folio_queue *folioq, unsigned int rreq_id)
 54{
 55	folio_batch_init(&folioq->vec);
 56	folioq->next = NULL;
 57	folioq->prev = NULL;
 58	folioq->marks = 0;
 59	folioq->marks2 = 0;
 60	folioq->rreq_id = rreq_id;
 61	folioq->debug_id = 0;
 62}
 63
 64/**
 65 * folioq_nr_slots: Query the capacity of a folio queue segment
 66 * @folioq: The segment to query
 67 *
 68 * Query the number of folios that a particular folio queue segment might hold.
 69 * [!] NOTE: This must not be assumed to be the same for every segment!
 70 */
 71static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
 72{
 73	return PAGEVEC_SIZE;
 74}
 75
 76/**
 77 * folioq_count: Query the occupancy of a folio queue segment
 78 * @folioq: The segment to query
 79 *
 80 * Query the number of folios that have been added to a folio queue segment.
 81 * Note that this is not decreased as folios are removed from a segment.
 82 */
 83static inline unsigned int folioq_count(struct folio_queue *folioq)
 84{
 85	return folio_batch_count(&folioq->vec);
 86}
 87
 88/**
 89 * folioq_full: Query if a folio queue segment is full
 90 * @folioq: The segment to query
 91 *
 92 * Query if a folio queue segment is fully occupied.  Note that this does not
 93 * change if folios are removed from a segment.
 94 */
 95static inline bool folioq_full(struct folio_queue *folioq)
 96{
 97	//return !folio_batch_space(&folioq->vec);
 98	return folioq_count(folioq) >= folioq_nr_slots(folioq);
 99}
100
101/**
102 * folioq_is_marked: Check first folio mark in a folio queue segment
103 * @folioq: The segment to query
104 * @slot: The slot number of the folio to query
105 *
106 * Determine if the first mark is set for the folio in the specified slot in a
107 * folio queue segment.
108 */
109static inline bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot)
110{
111	return test_bit(slot, &folioq->marks);
112}
113
114/**
115 * folioq_mark: Set the first mark on a folio in a folio queue segment
116 * @folioq: The segment to modify
117 * @slot: The slot number of the folio to modify
118 *
119 * Set the first mark for the folio in the specified slot in a folio queue
120 * segment.
121 */
122static inline void folioq_mark(struct folio_queue *folioq, unsigned int slot)
123{
124	set_bit(slot, &folioq->marks);
125}
126
127/**
128 * folioq_unmark: Clear the first mark on a folio in a folio queue segment
129 * @folioq: The segment to modify
130 * @slot: The slot number of the folio to modify
131 *
132 * Clear the first mark for the folio in the specified slot in a folio queue
133 * segment.
134 */
135static inline void folioq_unmark(struct folio_queue *folioq, unsigned int slot)
136{
137	clear_bit(slot, &folioq->marks);
138}
139
140/**
141 * folioq_is_marked2: Check second folio mark in a folio queue segment
142 * @folioq: The segment to query
143 * @slot: The slot number of the folio to query
144 *
145 * Determine if the second mark is set for the folio in the specified slot in a
146 * folio queue segment.
147 */
148static inline bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot)
149{
150	return test_bit(slot, &folioq->marks2);
151}
152
153/**
154 * folioq_mark2: Set the second mark on a folio in a folio queue segment
155 * @folioq: The segment to modify
156 * @slot: The slot number of the folio to modify
157 *
158 * Set the second mark for the folio in the specified slot in a folio queue
159 * segment.
160 */
161static inline void folioq_mark2(struct folio_queue *folioq, unsigned int slot)
162{
163	set_bit(slot, &folioq->marks2);
164}
165
166/**
167 * folioq_unmark2: Clear the second mark on a folio in a folio queue segment
168 * @folioq: The segment to modify
169 * @slot: The slot number of the folio to modify
170 *
171 * Clear the second mark for the folio in the specified slot in a folio queue
172 * segment.
173 */
174static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot)
175{
176	clear_bit(slot, &folioq->marks2);
177}
178
179/**
180 * folioq_append: Add a folio to a folio queue segment
181 * @folioq: The segment to add to
182 * @folio: The folio to add
183 *
184 * Add a folio to the tail of the sequence in a folio queue segment, increasing
185 * the occupancy count and returning the slot number for the folio just added.
186 * The folio size is extracted and stored in the queue and the marks are left
187 * unmodified.
188 *
189 * Note that it's left up to the caller to check that the segment capacity will
190 * not be exceeded and to extend the queue.
191 */
192static inline unsigned int folioq_append(struct folio_queue *folioq, struct folio *folio)
193{
194	unsigned int slot = folioq->vec.nr++;
195
196	folioq->vec.folios[slot] = folio;
197	folioq->orders[slot] = folio_order(folio);
198	return slot;
199}
200
201/**
202 * folioq_append_mark: Add a folio to a folio queue segment
203 * @folioq: The segment to add to
204 * @folio: The folio to add
205 *
206 * Add a folio to the tail of the sequence in a folio queue segment, increasing
207 * the occupancy count and returning the slot number for the folio just added.
208 * The folio size is extracted and stored in the queue, the first mark is set
209 * and and the second and third marks are left unmodified.
210 *
211 * Note that it's left up to the caller to check that the segment capacity will
212 * not be exceeded and to extend the queue.
213 */
214static inline unsigned int folioq_append_mark(struct folio_queue *folioq, struct folio *folio)
215{
216	unsigned int slot = folioq->vec.nr++;
217
218	folioq->vec.folios[slot] = folio;
219	folioq->orders[slot] = folio_order(folio);
220	folioq_mark(folioq, slot);
221	return slot;
222}
223
224/**
225 * folioq_folio: Get a folio from a folio queue segment
226 * @folioq: The segment to access
227 * @slot: The folio slot to access
228 *
229 * Retrieve the folio in the specified slot from a folio queue segment.  Note
230 * that no bounds check is made and if the slot hasn't been added into yet, the
231 * pointer will be undefined.  If the slot has been cleared, NULL will be
232 * returned.
233 */
234static inline struct folio *folioq_folio(const struct folio_queue *folioq, unsigned int slot)
235{
236	return folioq->vec.folios[slot];
237}
238
239/**
240 * folioq_folio_order: Get the order of a folio from a folio queue segment
241 * @folioq: The segment to access
242 * @slot: The folio slot to access
243 *
244 * Retrieve the order of the folio in the specified slot from a folio queue
245 * segment.  Note that no bounds check is made and if the slot hasn't been
246 * added into yet, the order returned will be 0.
247 */
248static inline unsigned int folioq_folio_order(const struct folio_queue *folioq, unsigned int slot)
249{
250	return folioq->orders[slot];
251}
252
253/**
254 * folioq_folio_size: Get the size of a folio from a folio queue segment
255 * @folioq: The segment to access
256 * @slot: The folio slot to access
257 *
258 * Retrieve the size of the folio in the specified slot from a folio queue
259 * segment.  Note that no bounds check is made and if the slot hasn't been
260 * added into yet, the size returned will be PAGE_SIZE.
261 */
262static inline size_t folioq_folio_size(const struct folio_queue *folioq, unsigned int slot)
263{
264	return PAGE_SIZE << folioq_folio_order(folioq, slot);
265}
266
267/**
268 * folioq_clear: Clear a folio from a folio queue segment
269 * @folioq: The segment to clear
270 * @slot: The folio slot to clear
271 *
272 * Clear a folio from a sequence in a folio queue segment and clear its marks.
273 * The occupancy count is left unchanged.
274 */
275static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot)
276{
277	folioq->vec.folios[slot] = NULL;
278	folioq_unmark(folioq, slot);
279	folioq_unmark2(folioq, slot);
280}
281
282#endif /* _LINUX_FOLIO_QUEUE_H */