mp_init.c File Reference

#include <console/console.h>
#include <string.h>
#include <rmodule.h>
#include <commonlib/helpers.h>
#include <cpu/cpu.h>
#include <cpu/intel/microcode.h>
#include <cpu/x86/cache.h>
#include <cpu/x86/gdt.h>
#include <cpu/x86/lapic.h>
#include <cpu/x86/name.h>
#include <cpu/x86/msr.h>
#include <cpu/x86/mtrr.h>
#include <cpu/x86/smm.h>
#include <cpu/x86/mp.h>
#include <delay.h>
#include <device/device.h>
#include <device/path.h>
#include <smp/atomic.h>
#include <smp/spinlock.h>
#include <symbols.h>
#include <timer.h>
#include <thread.h>
#include <types.h>
#include <security/intel/stm/SmmStm.h>

Include dependency graph for mp_init.c:

Go to the source code of this file.

Data Structures
struct	mp_callback
struct	mp_flight_record
struct	mp_params
struct	sipi_params
struct	saved_msr
struct	mp_flight_plan
struct	mp_state

Macros
#define	MAX_APIC_IDS   256
#define	_MP_FLIGHT_RECORD(barrier_, ap_func_, bsp_func_)
#define	MP_FR_BLOCK_APS(ap_func_, bsp_func_)    _MP_FLIGHT_RECORD(0, ap_func_, bsp_func_)
#define	MP_FR_NOBLOCK_APS(ap_func_, bsp_func_)    _MP_FLIGHT_RECORD(1, ap_func_, bsp_func_)
#define	NUM_FIXED_MTRRS   11

Functions
struct mp_flight_record	__aligned (CACHELINE_SIZE)
static void	barrier_wait (atomic_t *b)
static void	release_barrier (atomic_t *b)
static enum cb_err	wait_for_aps (atomic_t *val, int target, int total_delay, int delay_step)
static void	ap_do_flight_plan (void)
static void	park_this_cpu (void *unused)
static void asmlinkage	ap_init (void)
static void	setup_default_sipi_vector_params (struct sipi_params *sp)
static struct saved_msr *	save_msr (int index, struct saved_msr *entry)
static int	save_bsp_msrs (char *start, int size)
static atomic_t *	load_sipi_vector (struct mp_params *mp_params)
static int	allocate_cpu_devices (struct bus *cpu_bus, struct mp_params *p)
static enum cb_err	apic_wait_timeout (int total_delay, int delay_step)
static enum cb_err	send_sipi_to_aps (int ap_count, atomic_t *num_aps, int sipi_vector)
static enum cb_err	start_aps (struct bus *cpu_bus, int ap_count, atomic_t *num_aps)
static enum cb_err	bsp_do_flight_plan (struct mp_params *mp_params)
static void	init_bsp (struct bus *cpu_bus)
static enum cb_err	mp_init (struct bus *cpu_bus, struct mp_params *p)
static void	mp_initialize_cpu (void)
void	smm_initiate_relocation_parallel (void)
	DECLARE_SPIN_LOCK (smm_relocation_lock)
void	smm_initiate_relocation (void)
static int	is_smm_enabled (void)
static void	smm_disable (void)
static void	smm_enable (void)
static void asmlinkage	smm_do_relocation (void *arg)
static void	adjust_smm_apic_id_map (struct smm_loader_params *smm_params)
static enum cb_err	install_relocation_handler (int num_cpus, size_t real_save_state_size, size_t save_state_size)
static enum cb_err	install_permanent_handler (int num_cpus, uintptr_t smbase, size_t smsize, size_t real_save_state_size, size_t save_state_size)
static void	load_smm_handlers (void)
static void	trigger_smm_relocation (void)
static struct mp_callback *	read_callback (struct mp_callback **slot)
static void	store_callback (struct mp_callback **slot, struct mp_callback *val)
static enum cb_err	run_ap_work (struct mp_callback *val, long expire_us)
static void	ap_wait_for_instruction (void)
enum cb_err	mp_run_on_aps (void(*func)(void *), void *arg, int logical_cpu_num, long expire_us)
enum cb_err	mp_run_on_all_aps (void(*func)(void *), void *arg, long expire_us, bool run_parallel)
enum cb_err	mp_run_on_all_cpus (void(*func)(void *), void *arg)
enum cb_err	mp_park_aps (void)
static size_t	smm_stub_size (void)
static void	fill_mp_state_smm (struct mp_state *state, const struct mp_ops *ops)
static void	fill_mp_state (struct mp_state *state, const struct mp_ops *ops)
static enum cb_err	do_mp_init_with_smm (struct bus *cpu_bus, const struct mp_ops *mp_ops)
enum cb_err	mp_init_with_smm (struct bus *cpu_bus, const struct mp_ops *mp_ops)

Variables
static char	processor_name [49]
atomic_t	barrier
atomic_t	cpus_entered
void(*	ap_call )(void)
void(*	bsp_call )(void)
struct mp_params	__aligned
struct sipi_params	__packed
char	_binary_sipi_vector_start []
static const uintptr_t	sipi_vector_location = SMM_DEFAULT_BASE
static const int	sipi_vector_location_size = SMM_DEFAULT_SIZE
static int	global_num_aps
static struct mp_flight_plan	mp_info
static struct device *	cpus_dev [CONFIG_MAX_CPUS]
static const unsigned int	fixed_mtrrs [NUM_FIXED_MTRRS]
struct mp_state	mp_state
static struct mp_callback *	ap_callbacks [CONFIG_MAX_CPUS]
static struct mp_flight_record	mp_steps []

Macro Definition Documentation

_MP_FLIGHT_RECORD

#define _MP_FLIGHT_RECORD	(		barrier_,
			ap_func_,
			bsp_func_
	)

Value:

        {                                                       \
                .barrier = ATOMIC_INIT(barrier_),               \
                .cpus_entered = ATOMIC_INIT(0),                 \
                .ap_call = ap_func_,                            \
                .bsp_call = bsp_func_,                          \
        }

Definition at line 59 of file mp_init.c.

MAX_APIC_IDS

#define MAX_APIC_IDS   256

Definition at line 29 of file mp_init.c.

MP_FR_BLOCK_APS

#define MP_FR_BLOCK_APS	(		ap_func_,
			bsp_func_
	)		_MP_FLIGHT_RECORD(0, ap_func_, bsp_func_)

Definition at line 67 of file mp_init.c.

MP_FR_NOBLOCK_APS

#define MP_FR_NOBLOCK_APS	(		ap_func_,
			bsp_func_
	)		_MP_FLIGHT_RECORD(1, ap_func_, bsp_func_)

Definition at line 70 of file mp_init.c.

NUM_FIXED_MTRRS

#define NUM_FIXED_MTRRS   11

Definition at line 221 of file mp_init.c.

Function Documentation

__aligned()

struct mp_flight_record __aligned

(

CACHELINE_SIZE

)

adjust_smm_apic_id_map()

static void adjust_smm_apic_id_map ( struct smm_loader_params *  smm_params )

static

Definition at line 750 of file mp_init.c.

References smm_stub_params::apic_id_to_cpu, cpu_get_apic_id(), and smm_loader_params::stub_params.

Here is the call graph for this function:

allocate_cpu_devices()

static int allocate_cpu_devices ( struct bus *  cpu_bus, struct mp_params *  p  )

static

Definition at line 356 of file mp_init.c.

References alloc_find_dev(), BIOS_CRIT, cpu_info(), cpus_dev, DEVICE_PATH_APIC, info, NULL, mp_params::num_cpus, printk, and processor_name.

Here is the call graph for this function:

ap_do_flight_plan()

static void ap_do_flight_plan ( void  )

static

Definition at line 159 of file mp_init.c.

References mp_flight_record::ap_call, atomic_inc, mp_flight_record::barrier, barrier_wait(), mp_flight_record::cpus_entered, mp_info, NULL, mp_flight_plan::num_records, and mp_flight_plan::records.

Referenced by ap_init().

Here is the call graph for this function:

Here is the caller graph for this function:

ap_init()

static void asmlinkage ap_init ( void  )

static

Definition at line 181 of file mp_init.c.

References ap_do_flight_plan(), BIOS_INFO, cpu_add_map_entry(), cpu_info(), cpu_is_intel(), cpus_dev, enable_lapic(), get_current_microcode_rev(), info, lapicid(), NULL, park_this_cpu(), printk, and setup_lapic_interrupts().

Referenced by load_sipi_vector().

Here is the call graph for this function:

Here is the caller graph for this function:

ap_wait_for_instruction()

static void ap_wait_for_instruction ( void  )

static

Definition at line 933 of file mp_init.c.

References ap_callbacks, mp_callback::arg, BIOS_ERR, CONFIG, cpu_index(), mp_callback::func, mp_callback::logical_cpu_number, memcpy(), mfence(), NULL, printk, read_callback(), and store_callback().

Here is the call graph for this function:

apic_wait_timeout()

static enum cb_err apic_wait_timeout ( int  total_delay, int  delay_step  )

static

Definition at line 356 of file mp_init.c.

Referenced by smm_initiate_relocation_parallel().

Here is the caller graph for this function:

barrier_wait()

static void barrier_wait ( atomic_t *  b )

inlinestatic

Definition at line 129 of file mp_init.c.

References atomic_read, and mfence().

Referenced by ap_do_flight_plan().

Here is the call graph for this function:

Here is the caller graph for this function:

bsp_do_flight_plan()

static enum cb_err bsp_do_flight_plan ( struct mp_params *  mp_params )

static

Definition at line 356 of file mp_init.c.

DECLARE_SPIN_LOCK()

DECLARE_SPIN_LOCK

(

smm_relocation_lock

)

do_mp_init_with_smm()

static enum cb_err do_mp_init_with_smm ( struct bus *  cpu_bus, const struct mp_ops *  mp_ops  )

static

Definition at line 1082 of file mp_init.c.

References CONFIG, fill_mp_state_smm(), memcpy(), NULL, and ops.

Here is the call graph for this function:

fill_mp_state()

static void fill_mp_state ( struct mp_state *  state, const struct mp_ops *  ops  )

static

Definition at line 1082 of file mp_init.c.

fill_mp_state_smm()

static void fill_mp_state_smm ( struct mp_state *  state, const struct mp_ops *  ops  )

static

Definition at line 1058 of file mp_init.c.

References ALIGN_UP, CONFIG, MAX, NULL, mp_state::ops, ops, mp_ops::per_cpu_smm_trigger, smm_initiate_relocation(), and smm_stub_size().

Referenced by do_mp_init_with_smm().

Here is the call graph for this function:

Here is the caller graph for this function:

init_bsp()

static void init_bsp ( struct bus *  cpu_bus )

static

Definition at line 533 of file mp_init.c.

References alloc_find_dev(), BIOS_CRIT, BIOS_INFO, cpu_add_map_entry(), cpu_info(), DEVICE_PATH_APIC, enable_lapic(), fill_processor_name(), info, lapicid(), printk, processor_name, and setup_lapic_interrupts().

Here is the call graph for this function:

install_permanent_handler()

static enum cb_err install_permanent_handler ( int  num_cpus, uintptr_t  smbase, size_t  smsize, size_t  real_save_state_size, size_t  save_state_size  )

static

Definition at line 750 of file mp_init.c.

Referenced by load_smm_handlers().

Here is the caller graph for this function:

install_relocation_handler()

static enum cb_err install_relocation_handler ( int  num_cpus, size_t  real_save_state_size, size_t  save_state_size  )

static

Definition at line 750 of file mp_init.c.

Referenced by load_smm_handlers().

Here is the caller graph for this function:

is_smm_enabled()

static int is_smm_enabled ( void  )

static

Definition at line 684 of file mp_init.c.

References CONFIG, and mp_state::do_smm.

Referenced by load_smm_handlers(), and trigger_smm_relocation().

Here is the caller graph for this function:

load_sipi_vector()

static atomic_t* load_sipi_vector ( struct mp_params *  mp_params )

static

Definition at line 282 of file mp_init.c.

References _binary_sipi_vector_start, ALIGN_UP, sipi_params::ap_count, ap_init(), atomic_set, BIOS_CRIT, sipi_params::c_handler, sipi_params::microcode_lock, mp_params::microcode_pointer, sipi_params::microcode_ptr, sipi_params::msr_count, sipi_params::msr_table_ptr, NULL, mp_params::parallel_microcode_load, printk, rmodule_entry_offset(), rmodule_load(), rmodule_load_alignment(), rmodule_memory_size(), rmodule_parameters(), rmodule_parse(), save_bsp_msrs(), setup_default_sipi_vector_params(), sipi_vector_location, and sipi_vector_location_size.

Here is the call graph for this function:

load_smm_handlers()

static void load_smm_handlers ( void  )

static

Definition at line 810 of file mp_init.c.

References BIOS_ERR, CB_SUCCESS, mp_state::cpu_count, install_permanent_handler(), install_relocation_handler(), is_smm_enabled(), NULL, mp_state::ops, mp_state::perm_smbase, mp_state::perm_smsize, mp_ops::pre_mp_smm_init, printk, smm_disable(), mp_state::smm_real_save_state_size, mp_state::smm_save_state_size, smm_setup_stack(), and wbinvd().

Here is the call graph for this function:

mp_init()

static enum cb_err mp_init ( struct bus *  cpu_bus, struct mp_params *  p  )

static

Definition at line 533 of file mp_init.c.

mp_init_with_smm()

enum cb_err mp_init_with_smm	(	struct bus *	cpu_bus,
		const struct mp_ops *	mp_ops
	)

Definition at line 1082 of file mp_init.c.

Referenced by cpx_init_cpus(), get_adl_cpu_type(), mp_init_cpus(), soc_init_cpus(), and xeon_sp_init_cpus().

Here is the caller graph for this function:

mp_initialize_cpu()

static void mp_initialize_cpu ( void  )

static

Definition at line 632 of file mp_init.c.

References cpu_info(), cpu_initialize(), and info.

Here is the call graph for this function:

mp_park_aps()

enum cb_err mp_park_aps

(

void

)

Definition at line 933 of file mp_init.c.

Referenced by arch_bootstate_coreboot_exit(), and intel_txt_prepare_txt_env().

Here is the caller graph for this function:

mp_run_on_all_aps()

enum cb_err mp_run_on_all_aps	(	void(*)(void *)	func,
		void *	arg,
		long	expire_us,
		bool	run_parallel
	)

Definition at line 933 of file mp_init.c.

Referenced by mp_startup_all_aps(), and mp_startup_all_cpus().

Here is the caller graph for this function:

mp_run_on_all_cpus()

enum cb_err mp_run_on_all_cpus	(	void(*)(void *)	func,
		void *	arg
	)

Definition at line 933 of file mp_init.c.

Referenced by drop_privilege_all(), finalize_cores(), post_cpus_init(), post_mp_init(), and run_set_cpu_type().

Here is the caller graph for this function:

mp_run_on_aps()

enum cb_err mp_run_on_aps	(	void(*)(void *)	func,
		void *	arg,
		int	logical_cpu_num,
		long	expire_us
	)

Definition at line 933 of file mp_init.c.

Referenced by agesa_RunFcnOnAllAps(), agesa_RunFuncOnAp(), mp_startup_this_ap(), and ocp_oem_smbios_strings().

Here is the caller graph for this function:

park_this_cpu()

static void park_this_cpu ( void *  unused )

static

Definition at line 174 of file mp_init.c.

References stop_this_cpu().

Referenced by ap_init().

Here is the call graph for this function:

Here is the caller graph for this function:

read_callback()

static struct mp_callback* read_callback ( struct mp_callback **  slot )

static

Definition at line 862 of file mp_init.c.

Referenced by ap_wait_for_instruction().

Here is the caller graph for this function:

release_barrier()

static void release_barrier ( atomic_t *  b )

inlinestatic

Definition at line 136 of file mp_init.c.

References atomic_set, and mfence().

Here is the call graph for this function:

run_ap_work()

static enum cb_err run_ap_work ( struct mp_callback *  val, long  expire_us  )

static

Definition at line 874 of file mp_init.c.

References val.

save_bsp_msrs()

static int save_bsp_msrs ( char *  start, int  size  )

static

Definition at line 243 of file mp_init.c.

References BIOS_CRIT, fixed_mtrrs, fixed_mtrrs_expose_amd_rwdram(), fixed_mtrrs_hide_amd_rwdram(), get_var_mtrr_count(), MTRR_DEF_TYPE_MSR, MTRR_PHYS_BASE, MTRR_PHYS_MASK, NUM_FIXED_MTRRS, printk, save_msr(), and void().

Referenced by load_sipi_vector().

Here is the call graph for this function:

Here is the caller graph for this function:

save_msr()

static struct saved_msr* save_msr ( int  index, struct saved_msr *  entry  )

inlinestatic

Definition at line 229 of file mp_init.c.

References saved_msr::hi, msr_struct::hi, saved_msr::index, saved_msr::lo, msr_struct::lo, and rdmsr().

Referenced by save_bsp_msrs().

Here is the call graph for this function:

Here is the caller graph for this function:

send_sipi_to_aps()

static enum cb_err send_sipi_to_aps ( int  ap_count, atomic_t *  num_aps, int  sipi_vector  )

static

Definition at line 356 of file mp_init.c.

setup_default_sipi_vector_params()

static void setup_default_sipi_vector_params ( struct sipi_params *  sp )

static

Definition at line 210 of file mp_init.c.

References ALIGN_DOWN, sipi_params::gdt, gdt, gdt_end, sipi_params::gdtlimit, sipi_params::idt_ptr, idtarg, sipi_params::per_cpu_segment_descriptors, per_cpu_segment_descriptors, sipi_params::per_cpu_segment_selector, per_cpu_segment_selector, sipi_params::stack_size, and sipi_params::stack_top.

Referenced by load_sipi_vector().

Here is the caller graph for this function:

smm_disable()

static void smm_disable ( void  )

static

Definition at line 689 of file mp_init.c.

References mp_state::do_smm.

Referenced by load_smm_handlers().

Here is the caller graph for this function:

smm_do_relocation()

static void asmlinkage smm_do_relocation ( void *  arg )

static

Definition at line 705 of file mp_init.c.

References arg, BIOS_CRIT, BIOS_ERR, BIOS_INFO, CONFIG, smm_module_params::cpu, mp_state::ops, mp_state::perm_smbase, mp_state::perm_smsize, printk, mp_state::reloc_start32_offset, mp_ops::relocation_handler, SMM_DEFAULT_BASE, smm_get_cpu_smbase(), and stm_setup().

Here is the call graph for this function:

smm_enable()

static void smm_enable ( void  )

static

Definition at line 694 of file mp_init.c.

References CONFIG, and mp_state::do_smm.

smm_initiate_relocation()

void smm_initiate_relocation

(

void

)

Definition at line 664 of file mp_init.c.

References smm_initiate_relocation_parallel(), spin_lock, and spin_unlock.

Referenced by fill_mp_state_smm(), per_cpu_smm_trigger(), smm_initialize(), and smm_relocate().

Here is the call graph for this function:

Here is the caller graph for this function:

smm_initiate_relocation_parallel()

void smm_initiate_relocation_parallel

(

void

)

Definition at line 639 of file mp_init.c.

References apic_wait_timeout(), BIOS_DEBUG, CB_SUCCESS, lapic_busy(), LAPIC_DM_SMI, LAPIC_INT_ASSERT, lapic_send_ipi_self(), and printk.

Referenced by smm_initiate_relocation(), and smm_relocate().

Here is the call graph for this function:

Here is the caller graph for this function:

smm_stub_size()

static size_t smm_stub_size ( void  )

static

Definition at line 1045 of file mp_init.c.

References _binary_smmstub_start, BIOS_ERR, printk, rmodule_memory_size(), and rmodule_parse().

Referenced by fill_mp_state_smm(), and smm_module_setup_stub().

Here is the call graph for this function:

Here is the caller graph for this function:

start_aps()

static enum cb_err start_aps ( struct bus *  cpu_bus, int  ap_count, atomic_t *  num_aps  )

static

Definition at line 356 of file mp_init.c.

store_callback()

static void store_callback ( struct mp_callback **  slot, struct mp_callback *  val  )

static

Definition at line 874 of file mp_init.c.

Referenced by ap_wait_for_instruction().

Here is the caller graph for this function:

trigger_smm_relocation()

static void trigger_smm_relocation ( void  )

static

Definition at line 851 of file mp_init.c.

References is_smm_enabled(), NULL, mp_state::ops, and mp_ops::per_cpu_smm_trigger.

Here is the call graph for this function:

wait_for_aps()

static enum cb_err wait_for_aps ( atomic_t *  val, int  target, int  total_delay, int  delay_step  )

static

Definition at line 136 of file mp_init.c.

Variable Documentation

__aligned

struct mp_params __aligned

__packed

struct saved_msr __packed

_binary_sipi_vector_start

char _binary_sipi_vector_start[]

extern

Referenced by load_sipi_vector().

ap_call

void(* ap_call) (void)

(

void

)

Definition at line 2 of file mp_init.c.

ap_callbacks

struct mp_callback* ap_callbacks[CONFIG_MAX_CPUS]

static

Definition at line 860 of file mp_init.c.

Referenced by ap_wait_for_instruction().

barrier

atomic_t barrier

(

void

)

Definition at line 0 of file mp_init.c.

Referenced by smp_pause(), spinlock_lock(), and spinlock_unlock().

bsp_call

void(* bsp_call) (void)

(

void

)

Definition at line 3 of file mp_init.c.

cpus_dev

struct device* cpus_dev[CONFIG_MAX_CPUS]

static

Definition at line 127 of file mp_init.c.

Referenced by allocate_cpu_devices(), and ap_init().

cpus_entered

atomic_t cpus_entered

Definition at line 1 of file mp_init.c.

fixed_mtrrs

const unsigned int fixed_mtrrs[NUM_FIXED_MTRRS]

static

Initial value:

= {
        MTRR_FIX_64K_00000, MTRR_FIX_16K_80000, MTRR_FIX_16K_A0000,
        MTRR_FIX_4K_C0000, MTRR_FIX_4K_C8000, MTRR_FIX_4K_D0000,
        MTRR_FIX_4K_D8000, MTRR_FIX_4K_E0000, MTRR_FIX_4K_E8000,
        MTRR_FIX_4K_F0000, MTRR_FIX_4K_F8000,
}

Definition at line 222 of file mp_init.c.

Referenced by save_bsp_msrs().

global_num_aps

int global_num_aps

static

Definition at line 123 of file mp_init.c.

mp_info

struct mp_flight_plan mp_info

static

Definition at line 123 of file mp_init.c.

Referenced by ap_do_flight_plan().

mp_state

struct mp_state mp_state

mp_steps

struct mp_flight_record mp_steps[]

static

Initial value:

= {
        
        MP_FR_BLOCK_APS(NULL, load_smm_handlers),
        
        MP_FR_NOBLOCK_APS(trigger_smm_relocation, trigger_smm_relocation),
        
        MP_FR_BLOCK_APS(mp_initialize_cpu, mp_initialize_cpu),
        
        MP_FR_BLOCK_APS(ap_wait_for_instruction, NULL),
}

Definition at line 933 of file mp_init.c.

processor_name

char processor_name[49]

static

Definition at line 37 of file mp_init.c.

Referenced by allocate_cpu_devices(), fill_processor_name(), init_bsp(), model_2065x_init(), and model_206ax_report().

sipi_vector_location

const uintptr_t sipi_vector_location = SMM_DEFAULT_BASE

static

Definition at line 115 of file mp_init.c.

Referenced by load_sipi_vector().

sipi_vector_location_size

const int sipi_vector_location_size = SMM_DEFAULT_SIZE

static

Definition at line 116 of file mp_init.c.

Referenced by load_sipi_vector().