20 #include <smp/atomic.h>
21 #include <smp/spinlock.h>
29 #define MAX_APIC_IDS 256
59 #define _MP_FLIGHT_RECORD(barrier_, ap_func_, bsp_func_) \
61 .barrier = ATOMIC_INIT(barrier_), \
62 .cpus_entered = ATOMIC_INIT(0), \
63 .ap_call = ap_func_, \
64 .bsp_call = bsp_func_, \
67 #define MP_FR_BLOCK_APS(ap_func_, bsp_func_) \
68 _MP_FLIGHT_RECORD(0, ap_func_, bsp_func_)
70 #define MP_FR_NOBLOCK_APS(ap_func_, bsp_func_) \
71 _MP_FLIGHT_RECORD(1, ap_func_, bsp_func_)
148 delayed += delay_step;
149 if (delayed >= total_delay) {
201 info->cpu->path.apic.apic_id);
221 #define NUM_FIXED_MTRRS 11
256 if ((msr_count *
sizeof(
struct saved_msr)) > size) {
263 msr_entry = (
void *)start;
267 for (i = 0; i < num_var_mtrrs; i++) {
311 module_size =
ALIGN_UP(module_size, 4);
313 if (module_size > loc_size) {
315 module_size, loc_size);
319 num_msrs =
save_bsp_msrs(&mod_loc[module_size], loc_size - module_size);
363 if (max_cpus > CONFIG_MAX_CPUS) {
365 max_cpus, CONFIG_MAX_CPUS);
366 max_cpus = CONFIG_MAX_CPUS;
370 for (i = 1; i < max_cpus; i++) {
379 cpu_path.apic.apic_id =
info->cpu->path.apic.apic_id + i;
402 if (total >= total_delay) {
438 const int max_vector_loc = ((1 << 20) - (1 << 12)) >> 12;
446 if (sipi_vector > max_vector_loc) {
466 if (!
CONFIG(X86_INIT_NEED_1_SIPI)) {
503 const int step_us = 100;
555 if (
info->index != 0)
598 if (num_cpus < p->num_cpus) {
600 "ERROR: More cpus requested (%d) than supported (%d).\n",
611 if (ap_count ==
NULL)
696 if (
CONFIG(HAVE_SMI_HANDLER))
715 if (
cpu >= CONFIG_MAX_CPUS) {
717 "Invalid CPU number assigned in SMM stub: %d\n",
cpu);
733 printk(
BIOS_INFO,
"%s : curr_smbase 0x%x perm_smbase 0x%x, cpu = %d\n",
734 __func__, (
int)curr_smbase, (
int)perm_smbase,
cpu);
755 for (i = 0; i < CONFIG_MAX_CPUS; i++)
760 size_t save_state_size)
764 .real_cpu_save_state_size = real_save_state_size,
765 .per_cpu_save_state_size = save_state_size,
766 .num_concurrent_save_states = 1,
782 size_t smsize,
size_t real_save_state_size,
783 size_t save_state_size)
794 .real_cpu_save_state_size = real_save_state_size,
795 .per_cpu_save_state_size = save_state_size,
796 .num_concurrent_save_states =
num_cpus,
799 printk(
BIOS_DEBUG,
"Installing permanent SMM handler to 0x%08lx\n", smbase);
820 CONFIG_SMM_MODULE_STACK_SIZE)) {
866 asm volatile (
"mov %1, %0\n"
876 asm volatile (
"mov %1, %0\n"
890 if (!
CONFIG(PARALLEL_MP_AP_WORK)) {
891 printk(
BIOS_ERR,
"APs already parked. PARALLEL_MP_AP_WORK not selected.\n");
928 printk(
BIOS_CRIT,
"CRITICAL ERROR: AP call expired. %d/%d CPUs accepted.\n",
939 if (!
CONFIG(PARALLEL_MP_AP_WORK))
960 memcpy(&lcb, cb,
sizeof(lcb));
975 .logical_cpu_number = logical_cpu_num};
982 int ap_index, bsp_index;
991 for (ap_index = 0; ap_index < total_threads; ap_index++) {
993 if (ap_index == bsp_index)
1015 long duration_msecs;
1051 printk(
BIOS_ERR,
"%s: unable to get SMM module size\n", __func__);
1060 if (
ops->get_smm_info !=
NULL)
1062 &
state->smm_real_save_state_size);
1070 state->smm_save_state_size +=
1078 if (
ops->per_cpu_smm_trigger ==
NULL)
1090 if (
ops->get_cpu_count !=
NULL)
1091 state->cpu_count =
ops->get_cpu_count();
1093 if (
CONFIG(HAVE_SMI_HANDLER))
void stm_setup(uintptr_t mseg, int cpu, uintptr_t smbase, uintptr_t smbase_base, uint32_t offset32)
#define atomic_set(v, val)
int cpu_get_apic_id(int logical_cpu)
void cpu_initialize(unsigned int index)
void cpu_add_map_entry(unsigned int index)
static struct cpu_info * cpu_info(void)
static bool cpu_is_intel(void)
void * memcpy(void *dest, const void *src, size_t n)
void * memset(void *dstpp, int c, size_t len)
void restore_default_smm_area(void *smm_save_area)
void * backup_default_smm_area(void)
cb_err
coreboot error codes
@ CB_ERR
Generic error code.
@ CB_SUCCESS
Call completed successfully.
#define printk(level,...)
static struct saved_msr * save_msr(int index, struct saved_msr *entry)
struct sipi_params __packed
static char processor_name[49]
static atomic_t * load_sipi_vector(struct mp_params *mp_params)
static const uintptr_t sipi_vector_location
enum cb_err mp_park_aps(void)
static enum cb_err mp_init(struct bus *cpu_bus, struct mp_params *p)
static void smm_enable(void)
static void adjust_smm_apic_id_map(struct smm_loader_params *smm_params)
static const unsigned int fixed_mtrrs[NUM_FIXED_MTRRS]
#define MP_FR_BLOCK_APS(ap_func_, bsp_func_)
static struct device * cpus_dev[CONFIG_MAX_CPUS]
static int allocate_cpu_devices(struct bus *cpu_bus, struct mp_params *p)
static struct mp_flight_record mp_steps[]
static void release_barrier(atomic_t *b)
enum cb_err mp_run_on_all_aps(void(*func)(void *), void *arg, long expire_us, bool run_parallel)
static void load_smm_handlers(void)
enum cb_err mp_init_with_smm(struct bus *cpu_bus, const struct mp_ops *mp_ops)
static void smm_disable(void)
static enum cb_err send_sipi_to_aps(int ap_count, atomic_t *num_aps, int sipi_vector)
static void trigger_smm_relocation(void)
static void fill_mp_state_smm(struct mp_state *state, const struct mp_ops *ops)
DECLARE_SPIN_LOCK(smm_relocation_lock)
static void asmlinkage smm_do_relocation(void *arg)
static int global_num_aps
static enum cb_err do_mp_init_with_smm(struct bus *cpu_bus, const struct mp_ops *mp_ops)
static void mp_initialize_cpu(void)
static void setup_default_sipi_vector_params(struct sipi_params *sp)
static int save_bsp_msrs(char *start, int size)
char _binary_sipi_vector_start[]
static void store_callback(struct mp_callback **slot, struct mp_callback *val)
static void asmlinkage ap_init(void)
enum cb_err mp_run_on_all_cpus(void(*func)(void *), void *arg)
#define MP_FR_NOBLOCK_APS(ap_func_, bsp_func_)
static const int sipi_vector_location_size
enum cb_err mp_run_on_aps(void(*func)(void *), void *arg, int logical_cpu_num, long expire_us)
static size_t smm_stub_size(void)
static struct mp_flight_plan mp_info
static enum cb_err install_permanent_handler(int num_cpus, uintptr_t smbase, size_t smsize, size_t real_save_state_size, size_t save_state_size)
static void fill_mp_state(struct mp_state *state, const struct mp_ops *ops)
static void ap_wait_for_instruction(void)
static enum cb_err start_aps(struct bus *cpu_bus, int ap_count, atomic_t *num_aps)
static enum cb_err bsp_do_flight_plan(struct mp_params *mp_params)
static void ap_do_flight_plan(void)
static enum cb_err install_relocation_handler(int num_cpus, size_t real_save_state_size, size_t save_state_size)
static struct mp_callback * ap_callbacks[CONFIG_MAX_CPUS]
static struct mp_callback * read_callback(struct mp_callback **slot)
static enum cb_err apic_wait_timeout(int total_delay, int delay_step)
static enum cb_err wait_for_aps(atomic_t *val, int target, int total_delay, int delay_step)
struct mp_params __aligned
static void init_bsp(struct bus *cpu_bus)
static enum cb_err run_ap_work(struct mp_callback *val, long expire_us)
void smm_initiate_relocation(void)
static void barrier_wait(atomic_t *b)
static void park_this_cpu(void *unused)
static int is_smm_enabled(void)
void smm_initiate_relocation_parallel(void)
void fixed_mtrrs_hide_amd_rwdram(void)
void fixed_mtrrs_expose_amd_rwdram(void)
void mdelay(unsigned int msecs)
struct device * alloc_find_dev(struct bus *parent, struct device_path *path)
See if a device structure already exists and if not allocate it.
static struct smmstore_params_info info
char per_cpu_segment_descriptors[]
uint32_t per_cpu_segment_selector
static __always_inline msr_t rdmsr(unsigned int index)
#define atomic_inc(v)
atomic_inc - increment atomic variable
#define spin_unlock(lock)
static int stopwatch_expired(struct stopwatch *sw)
static void stopwatch_init(struct stopwatch *sw)
static long stopwatch_duration_msecs(struct stopwatch *sw)
static void stopwatch_init_usecs_expire(struct stopwatch *sw, long us)
static struct device_operations ops
void setup_lapic_interrupts(void)
static __always_inline unsigned int lapicid(void)
static __always_inline void lapic_send_ipi_others(uint32_t icrlow)
static __always_inline int lapic_busy(void)
static __always_inline void lapic_send_ipi_self(uint32_t icrlow)
void stop_this_cpu(void)
Normally this function is defined in lapic.h as an always inline function that just keeps the CPU in ...
#define BIOS_INFO
BIOS_INFO - Expected events.
#define BIOS_DEBUG
BIOS_DEBUG - Verbose output.
#define BIOS_CRIT
BIOS_CRIT - Recovery unlikely.
#define BIOS_ERR
BIOS_ERR - System in incomplete state.
uint32_t get_current_microcode_rev(void)
void fill_processor_name(char *processor_name)
int rmodule_entry_offset(const struct rmodule *m)
int rmodule_parse(void *ptr, struct rmodule *m)
void * rmodule_parameters(const struct rmodule *m)
int rmodule_memory_size(const struct rmodule *m)
int rmodule_load(void *loc, struct rmodule *m)
int rmodule_load_alignment(const struct rmodule *m)
unsigned char _binary_smmstub_start[]
int smm_setup_stack(const uintptr_t perm_smbase, const size_t perm_smram_size, const unsigned int total_cpus, const size_t stack_size)
int smm_setup_relocation_handler(struct smm_loader_params *params)
u32 smm_get_cpu_smbase(unsigned int cpu_num)
int smm_load_module(const uintptr_t smram_base, const size_t smram_size, struct smm_loader_params *params)
static void * default_smm_area
struct mp_flight_record * records
void(* pre_mp_init)(void)
void(* get_microcode_info)(const void **microcode, int *parallel)
void(* relocation_handler)(int cpu, uintptr_t curr_smbase, uintptr_t staggered_smbase)
void(* post_mp_init)(void)
void(* per_cpu_smm_trigger)(void)
void(* pre_mp_smm_init)(void)
int parallel_microcode_load
const void * microcode_pointer
struct mp_flight_record * flight_plan
size_t smm_save_state_size
size_t smm_real_save_state_size
uintptr_t reloc_start32_offset
uint32_t per_cpu_segment_selector
uint32_t per_cpu_segment_descriptors
struct smm_stub_params * stub_params
u16 apic_id_to_cpu[CONFIG_MAX_CPUS]
#define MTRR_FIX_4K_E0000
#define MTRR_FIX_64K_00000
#define MTRR_PHYS_BASE(reg)
#define MTRR_PHYS_MASK(reg)
static int get_var_mtrr_count(void)
#define MTRR_FIX_4K_D8000
#define MTRR_FIX_4K_C8000
#define MTRR_FIX_4K_E8000
#define MTRR_FIX_16K_A0000
#define MTRR_FIX_4K_D0000
#define MTRR_FIX_4K_F0000
#define MTRR_FIX_4K_C0000
#define MTRR_FIX_16K_80000
#define MTRR_FIX_4K_F8000
#define MTRR_DEF_TYPE_MSR
typedef void(X86APIP X86EMU_intrFuncs)(int num)