gsp.rs

  1// SPDX-License-Identifier: GPL-2.0
  2
  3use core::mem::size_of_val;
  4
  5use kernel::{
  6    device,
  7    dma::{
  8        DataDirection,
  9        DmaAddress, //
 10    },
 11    kvec,
 12    prelude::*,
 13    scatterlist::{
 14        Owned,
 15        SGTable, //
 16    },
 17};
 18
 19use crate::{
 20    dma::DmaObject,
 21    firmware::riscv::RiscvFirmware,
 22    gpu::{
 23        Architecture,
 24        Chipset, //
 25    },
 26    gsp::GSP_PAGE_SIZE,
 27    num::FromSafeCast,
 28};
 29
 30/// Ad-hoc and temporary module to extract sections from ELF images.
 31///
 32/// Some firmware images are currently packaged as ELF files, where sections names are used as keys
 33/// to specific and related bits of data. Future firmware versions are scheduled to move away from
 34/// that scheme before nova-core becomes stable, which means this module will eventually be
 35/// removed.
 36mod elf {
 37    use core::mem::size_of;
 38
 39    use kernel::bindings;
 40    use kernel::str::CStr;
 41    use kernel::transmute::FromBytes;
 42
 43    /// Newtype to provide a [`FromBytes`] implementation.
 44    #[repr(transparent)]
 45    struct Elf64Hdr(bindings::elf64_hdr);
 46    // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability.
 47    unsafe impl FromBytes for Elf64Hdr {}
 48
 49    #[repr(transparent)]
 50    struct Elf64SHdr(bindings::elf64_shdr);
 51    // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability.
 52    unsafe impl FromBytes for Elf64SHdr {}
 53
 54    /// Tries to extract section with name `name` from the ELF64 image `elf`, and returns it.
 55    pub(super) fn elf64_section<'a, 'b>(elf: &'a [u8], name: &'b str) -> Option<&'a [u8]> {
 56        let hdr = &elf
 57            .get(0..size_of::<bindings::elf64_hdr>())
 58            .and_then(Elf64Hdr::from_bytes)?
 59            .0;
 60
 61        // Get all the section headers.
 62        let mut shdr = {
 63            let shdr_num = usize::from(hdr.e_shnum);
 64            let shdr_start = usize::try_from(hdr.e_shoff).ok()?;
 65            let shdr_end = shdr_num
 66                .checked_mul(size_of::<Elf64SHdr>())
 67                .and_then(|v| v.checked_add(shdr_start))?;
 68
 69            elf.get(shdr_start..shdr_end)
 70                .map(|slice| slice.chunks_exact(size_of::<Elf64SHdr>()))?
 71        };
 72
 73        // Get the strings table.
 74        let strhdr = shdr
 75            .clone()
 76            .nth(usize::from(hdr.e_shstrndx))
 77            .and_then(Elf64SHdr::from_bytes)?;
 78
 79        // Find the section which name matches `name` and return it.
 80        shdr.find(|&sh| {
 81            let Some(hdr) = Elf64SHdr::from_bytes(sh) else {
 82                return false;
 83            };
 84
 85            let Some(name_idx) = strhdr
 86                .0
 87                .sh_offset
 88                .checked_add(u64::from(hdr.0.sh_name))
 89                .and_then(|idx| usize::try_from(idx).ok())
 90            else {
 91                return false;
 92            };
 93
 94            // Get the start of the name.
 95            elf.get(name_idx..)
 96                // Stop at the first `0`.
 97                .and_then(|nstr| nstr.get(0..=nstr.iter().position(|b| *b == 0)?))
 98                // Convert into CStr. This should never fail because of the line above.
 99                .and_then(|nstr| CStr::from_bytes_with_nul(nstr).ok())
100                // Convert into str.
101                .and_then(|c_str| c_str.to_str().ok())
102                // Check that the name matches.
103                .map(|str| str == name)
104                .unwrap_or(false)
105        })
106        // Return the slice containing the section.
107        .and_then(|sh| {
108            let hdr = Elf64SHdr::from_bytes(sh)?;
109            let start = usize::try_from(hdr.0.sh_offset).ok()?;
110            let end = usize::try_from(hdr.0.sh_size)
111                .ok()
112                .and_then(|sh_size| start.checked_add(sh_size))?;
113
114            elf.get(start..end)
115        })
116    }
117}
118
119/// GSP firmware with 3-level radix page tables for the GSP bootloader.
120///
121/// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address
122/// space:
123///
124/// ```text
125/// Level 0:  1 page, 1 entry         -> points to first level 1 page
126/// Level 1:  Multiple pages/entries  -> each entry points to a level 2 page
127/// Level 2:  Multiple pages/entries  -> each entry points to a firmware page
128/// ```
129///
130/// Each page is 4KB, each entry is 8 bytes (64-bit DMA address).
131/// Also known as "Radix3" firmware.
132#[pin_data]
133pub(crate) struct GspFirmware {
134    /// The GSP firmware inside a [`VVec`], device-mapped via a SG table.
135    #[pin]
136    fw: SGTable<Owned<VVec<u8>>>,
137    /// Level 2 page table whose entries contain DMA addresses of firmware pages.
138    #[pin]
139    level2: SGTable<Owned<VVec<u8>>>,
140    /// Level 1 page table whose entries contain DMA addresses of level 2 pages.
141    #[pin]
142    level1: SGTable<Owned<VVec<u8>>>,
143    /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page.
144    level0: DmaObject,
145    /// Size in bytes of the firmware contained in [`Self::fw`].
146    pub(crate) size: usize,
147    /// Device-mapped GSP signatures matching the GPU's [`Chipset`].
148    pub(crate) signatures: DmaObject,
149    /// GSP bootloader, verifies the GSP firmware before loading and running it.
150    pub(crate) bootloader: RiscvFirmware,
151}
152
153impl GspFirmware {
154    /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page
155    /// tables expected by the GSP bootloader to load it.
156    pub(crate) fn new<'a, 'b>(
157        dev: &'a device::Device<device::Bound>,
158        chipset: Chipset,
159        ver: &'b str,
160    ) -> Result<impl PinInit<Self, Error> + 'a> {
161        let fw = super::request_firmware(dev, chipset, "gsp", ver)?;
162
163        let fw_section = elf::elf64_section(fw.data(), ".fwimage").ok_or(EINVAL)?;
164
165        let sigs_section = match chipset.arch() {
166            Architecture::Ampere => ".fwsignature_ga10x",
167            Architecture::Ada => ".fwsignature_ad10x",
168            _ => return Err(ENOTSUPP),
169        };
170        let signatures = elf::elf64_section(fw.data(), sigs_section)
171            .ok_or(EINVAL)
172            .and_then(|data| DmaObject::from_data(dev, data))?;
173
174        let size = fw_section.len();
175
176        // Move the firmware into a vmalloc'd vector and map it into the device address
177        // space.
178        let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL)
179            .and_then(|mut v| {
180                v.extend_from_slice(fw_section, GFP_KERNEL)?;
181                Ok(v)
182            })
183            .map_err(|_| ENOMEM)?;
184
185        let bl = super::request_firmware(dev, chipset, "bootloader", ver)?;
186        let bootloader = RiscvFirmware::new(dev, &bl)?;
187
188        Ok(try_pin_init!(Self {
189            fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL),
190            level2 <- {
191                // Allocate the level 2 page table, map the firmware onto it, and map it into the
192                // device address space.
193                VVec::<u8>::with_capacity(
194                    fw.iter().count() * core::mem::size_of::<u64>(),
195                    GFP_KERNEL,
196                )
197                .map_err(|_| ENOMEM)
198                .and_then(|level2| map_into_lvl(&fw, level2))
199                .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))?
200            },
201            level1 <- {
202                // Allocate the level 1 page table, map the level 2 page table onto it, and map it
203                // into the device address space.
204                VVec::<u8>::with_capacity(
205                    level2.iter().count() * core::mem::size_of::<u64>(),
206                    GFP_KERNEL,
207                )
208                .map_err(|_| ENOMEM)
209                .and_then(|level1| map_into_lvl(&level2, level1))
210                .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))?
211            },
212            level0: {
213                // Allocate the level 0 page table as a device-visible DMA object, and map the
214                // level 1 page table onto it.
215
216                // Level 0 page table data.
217                let mut level0_data = kvec![0u8; GSP_PAGE_SIZE]?;
218
219                // Fill level 1 page entry.
220                let level1_entry = level1.iter().next().ok_or(EINVAL)?;
221                let level1_entry_addr = level1_entry.dma_address();
222                let dst = &mut level0_data[..size_of_val(&level1_entry_addr)];
223                dst.copy_from_slice(&level1_entry_addr.to_le_bytes());
224
225                // Turn the level0 page table into a [`DmaObject`].
226                DmaObject::from_data(dev, &level0_data)?
227            },
228            size,
229            signatures,
230            bootloader,
231        }))
232    }
233
234    /// Returns the DMA handle of the radix3 level 0 page table.
235    pub(crate) fn radix3_dma_handle(&self) -> DmaAddress {
236        self.level0.dma_handle()
237    }
238}
239
240/// Build a page table from a scatter-gather list.
241///
242/// Takes each DMA-mapped region from `sg_table` and writes page table entries
243/// for all 4KB pages within that region. For example, a 16KB SG entry becomes
244/// 4 consecutive page table entries.
245fn map_into_lvl(sg_table: &SGTable<Owned<VVec<u8>>>, mut dst: VVec<u8>) -> Result<VVec<u8>> {
246    for sg_entry in sg_table.iter() {
247        // Number of pages we need to map.
248        let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE);
249
250        for i in 0..num_pages {
251            let entry = sg_entry.dma_address()
252                + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE));
253            dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?;
254        }
255    }
256
257    Ok(dst)
258}