coreboot
coreboot is an Open Source project aimed at replacing the proprietary BIOS found in most computers.
raminit_common.c
Go to the documentation of this file.
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 
3 #include <assert.h>
4 #include <commonlib/helpers.h>
5 #include <console/console.h>
6 #include <cpu/intel/model_206ax/model_206ax.h>
7 #include <device/mmio.h>
8 #include <device/pci_ops.h>
9 #include <northbridge/intel/sandybridge/chip.h>
10 #include <device/pci_def.h>
11 #include <delay.h>
12 #include <types.h>
13 
14 #include "raminit_native.h"
15 #include "raminit_common.h"
16 #include "raminit_tables.h"
17 #include "sandybridge.h"
18 
19 /* FIXME: no support for 3-channel chipsets */
20 
21 static void sfence(void)
22 {
23  asm volatile ("sfence");
24 }
25 
26 /* Toggle IO reset bit */
27 static void toggle_io_reset(void)
28 {
29  u32 r32 = mchbar_read32(MC_INIT_STATE_G);
30  mchbar_write32(MC_INIT_STATE_G, r32 | (1 << 5));
31  udelay(1);
32  mchbar_write32(MC_INIT_STATE_G, r32 & ~(1 << 5));
33  udelay(1);
34 }
35 
36 static u32 get_XOVER_CLK(u8 rankmap)
37 {
38  return rankmap << 24;
39 }
40 
41 static u32 get_XOVER_CMD(u8 rankmap)
42 {
43  u32 reg;
44 
45  /* Enable xover cmd */
46  reg = 1 << 14;
47 
48  /* Enable xover ctl */
49  if (rankmap & 0x03)
50  reg |= (1 << 17);
51 
52  if (rankmap & 0x0c)
53  reg |= (1 << 26);
54 
55  return reg;
56 }
57 
58 void dram_find_common_params(ramctr_timing *ctrl)
59 {
60  size_t valid_dimms;
61  int channel, slot;
62  dimm_info *dimms = &ctrl->info;
63 
64  ctrl->cas_supported = (1 << (MAX_CAS - MIN_CAS + 1)) - 1;
65  valid_dimms = 0;
66 
67  FOR_ALL_CHANNELS for (slot = 0; slot < 2; slot++) {
68 
69  const struct dimm_attr_ddr3_st *dimm = &dimms->dimm[channel][slot];
70  if (dimm->dram_type != SPD_MEMORY_TYPE_SDRAM_DDR3)
71  continue;
72 
73  valid_dimms++;
74 
75  /* Find all possible CAS combinations */
76  ctrl->cas_supported &= dimm->cas_supported;
77 
78  /* Find the smallest common latencies supported by all DIMMs */
79  ctrl->tCK = MAX(ctrl->tCK, dimm->tCK);
80  ctrl->tAA = MAX(ctrl->tAA, dimm->tAA);
81  ctrl->tWR = MAX(ctrl->tWR, dimm->tWR);
82  ctrl->tRCD = MAX(ctrl->tRCD, dimm->tRCD);
83  ctrl->tRRD = MAX(ctrl->tRRD, dimm->tRRD);
84  ctrl->tRP = MAX(ctrl->tRP, dimm->tRP);
85  ctrl->tRAS = MAX(ctrl->tRAS, dimm->tRAS);
86  ctrl->tRFC = MAX(ctrl->tRFC, dimm->tRFC);
87  ctrl->tWTR = MAX(ctrl->tWTR, dimm->tWTR);
88  ctrl->tRTP = MAX(ctrl->tRTP, dimm->tRTP);
89  ctrl->tFAW = MAX(ctrl->tFAW, dimm->tFAW);
90  ctrl->tCWL = MAX(ctrl->tCWL, dimm->tCWL);
91  ctrl->tCMD = MAX(ctrl->tCMD, dimm->tCMD);
92  }
93 
94  if (!ctrl->cas_supported)
95  die("Unsupported DIMM combination. DIMMS do not support common CAS latency");
96 
97  if (!valid_dimms)
98  die("No valid DIMMs found");
99 }
100 
101 void dram_xover(ramctr_timing *ctrl)
102 {
103  u32 reg;
104  int channel;
105 
106  FOR_ALL_CHANNELS {
107  /* Enable xover clk */
108  reg = get_XOVER_CLK(ctrl->rankmap[channel]);
109  printram("XOVER CLK [%x] = %x\n", GDCRCKPICODE_ch(channel), reg);
110  mchbar_write32(GDCRCKPICODE_ch(channel), reg);
111 
112  /* Enable xover ctl & xover cmd */
113  reg = get_XOVER_CMD(ctrl->rankmap[channel]);
114  printram("XOVER CMD [%x] = %x\n", GDCRCMDPICODING_ch(channel), reg);
115  mchbar_write32(GDCRCMDPICODING_ch(channel), reg);
116  }
117 }
118 
119 static void dram_odt_stretch(ramctr_timing *ctrl, int channel)
120 {
121  u32 addr, stretch;
122 
123  stretch = ctrl->ref_card_offset[channel];
124  /*
125  * ODT stretch:
126  * Delay ODT signal by stretch value. Useful for multi DIMM setups on the same channel.
127  */
128  if (IS_SANDY_CPU(ctrl->cpu) && IS_SANDY_CPU_C(ctrl->cpu)) {
129  if (stretch == 2)
130  stretch = 3;
131 
132  addr = SCHED_SECOND_CBIT_ch(channel);
133  mchbar_clrsetbits32(addr, 0xf << 10, stretch << 12 | stretch << 10);
134  printk(RAM_DEBUG, "OTHP Workaround [%x] = %x\n", addr, mchbar_read32(addr));
135  } else {
136  addr = TC_OTHP_ch(channel);
137  union tc_othp_reg tc_othp = {
138  .raw = mchbar_read32(addr),
139  };
140  tc_othp.odt_delay_d0 = stretch;
141  tc_othp.odt_delay_d1 = stretch;
142  mchbar_write32(addr, tc_othp.raw);
143  printk(RAM_DEBUG, "OTHP [%x] = %x\n", addr, mchbar_read32(addr));
144  }
145 }
146 
147 void dram_timing_regs(ramctr_timing *ctrl)
148 {
149  int channel;
150 
151  /* BIN parameters */
152  const union tc_dbp_reg tc_dbp = {
153  .tRCD = ctrl->tRCD,
154  .tRP = ctrl->tRP,
155  .tAA = ctrl->CAS,
156  .tCWL = ctrl->CWL,
157  .tRAS = ctrl->tRAS,
158  };
159 
160  /* Regular access parameters */
161  const union tc_rap_reg tc_rap = {
162  .tRRD = ctrl->tRRD,
163  .tRTP = ctrl->tRTP,
164  .tCKE = ctrl->tCKE,
165  .tWTR = ctrl->tWTR,
166  .tFAW = ctrl->tFAW,
167  .tWR = ctrl->tWR,
168  .tCMD = 3,
169  };
170 
171  /* Other parameters */
172  const union tc_othp_reg tc_othp = {
173  .tXPDLL = MIN(ctrl->tXPDLL, 31),
174  .tXP = MIN(ctrl->tXP, 7),
175  .tAONPD = ctrl->tAONPD,
176  .tCPDED = 2,
177  .tPRPDEN = 1,
178  };
179 
180  /*
181  * If tXP and tXPDLL are very high, they no longer fit in the bitfields
182  * of the TC_OTHP register. If so, we set bits in TC_DTP to compensate.
183  * This can only happen on Ivy Bridge, and when overclocking the RAM.
184  */
185  const union tc_dtp_reg tc_dtp = {
186  .overclock_tXP = ctrl->tXP >= 8,
187  .overclock_tXPDLL = ctrl->tXPDLL >= 32,
188  };
189 
190  /*
191  * TC-Refresh timing parameters:
192  * The tREFIx9 field should be programmed to minimum of 8.9 * tREFI (to allow
193  * for possible delays from ZQ or isoc) and tRASmax (70us) divided by 1024.
194  */
195  const u32 val32 = MIN((ctrl->tREFI * 89) / 10, (70000 << 8) / ctrl->tCK);
196 
197  const union tc_rftp_reg tc_rftp = {
198  .tREFI = ctrl->tREFI,
199  .tRFC = ctrl->tRFC,
200  .tREFIx9 = val32 / 1024,
201  };
202 
203  /* Self-refresh timing parameters */
204  const union tc_srftp_reg tc_srftp = {
205  .tXSDLL = tDLLK,
206  .tXS_offset = ctrl->tXSOffset,
207  .tZQOPER = tDLLK - ctrl->tXSOffset,
208  .tMOD = ctrl->tMOD - 8,
209  };
210 
211  FOR_ALL_CHANNELS {
212  printram("DBP [%x] = %x\n", TC_DBP_ch(channel), tc_dbp.raw);
213  mchbar_write32(TC_DBP_ch(channel), tc_dbp.raw);
214 
215  printram("RAP [%x] = %x\n", TC_RAP_ch(channel), tc_rap.raw);
216  mchbar_write32(TC_RAP_ch(channel), tc_rap.raw);
217 
218  printram("OTHP [%x] = %x\n", TC_OTHP_ch(channel), tc_othp.raw);
219  mchbar_write32(TC_OTHP_ch(channel), tc_othp.raw);
220 
221  if (IS_IVY_CPU(ctrl->cpu)) {
222  /* Debug parameters - only applies to Ivy Bridge */
223  mchbar_write32(TC_DTP_ch(channel), tc_dtp.raw);
224  }
225 
226  dram_odt_stretch(ctrl, channel);
227 
228  printram("REFI [%x] = %x\n", TC_RFTP_ch(channel), tc_rftp.raw);
229  mchbar_write32(TC_RFTP_ch(channel), tc_rftp.raw);
230 
231  union tc_rfp_reg tc_rfp = {
232  .raw = mchbar_read32(TC_RFP_ch(channel)),
233  };
234  tc_rfp.oref_ri = 0xff;
235  mchbar_write32(TC_RFP_ch(channel), tc_rfp.raw);
236 
237  printram("SRFTP [%x] = %x\n", TC_SRFTP_ch(channel), tc_srftp.raw);
238  mchbar_write32(TC_SRFTP_ch(channel), tc_srftp.raw);
239  }
240 }
241 
242 void dram_dimm_mapping(ramctr_timing *ctrl)
243 {
244  int channel;
245  dimm_info *info = &ctrl->info;
246 
247  FOR_ALL_CHANNELS {
248  struct dimm_attr_ddr3_st *dimmA, *dimmB;
249  u32 reg = 0;
250 
251  if (info->dimm[channel][0].size_mb >= info->dimm[channel][1].size_mb) {
252  dimmA = &info->dimm[channel][0];
253  dimmB = &info->dimm[channel][1];
254  reg |= (0 << 16);
255  } else {
256  dimmA = &info->dimm[channel][1];
257  dimmB = &info->dimm[channel][0];
258  reg |= (1 << 16);
259  }
260 
261  if (dimmA && (dimmA->ranks > 0)) {
262  reg |= (dimmA->size_mb / 256) << 0;
263  reg |= (dimmA->ranks - 1) << 17;
264  reg |= (dimmA->width / 8 - 1) << 19;
265  }
266 
267  if (dimmB && (dimmB->ranks > 0)) {
268  reg |= (dimmB->size_mb / 256) << 8;
269  reg |= (dimmB->ranks - 1) << 18;
270  reg |= (dimmB->width / 8 - 1) << 20;
271  }
272 
273  /*
274  * Rank interleave: Bit 16 of the physical address space sets
275  * the rank to use in a dual single rank DIMM configuration.
276  * That results in every 64KiB being interleaved between two ranks.
277  */
278  reg |= 1 << 21;
279  /* Enhanced interleave */
280  reg |= 1 << 22;
281 
282  if ((dimmA && (dimmA->ranks > 0)) || (dimmB && (dimmB->ranks > 0))) {
283  ctrl->mad_dimm[channel] = reg;
284  } else {
285  ctrl->mad_dimm[channel] = 0;
286  }
287  }
288 }
289 
290 void dram_dimm_set_mapping(ramctr_timing *ctrl, int training)
291 {
292  int channel;
293  u32 ecc;
294 
295  if (ctrl->ecc_enabled)
296  ecc = training ? (1 << 24) : (3 << 24);
297  else
298  ecc = 0;
299 
300  FOR_ALL_CHANNELS {
301  mchbar_write32(MAD_DIMM(channel), ctrl->mad_dimm[channel] | ecc);
302  }
303 
304  if (ctrl->ecc_enabled)
305  udelay(10);
306 }
307 
308 void dram_zones(ramctr_timing *ctrl, int training)
309 {
310  u32 reg, ch0size, ch1size;
311  u8 val;
312  reg = 0;
313  val = 0;
314 
315  if (training) {
316  ch0size = ctrl->channel_size_mb[0] ? 256 : 0;
317  ch1size = ctrl->channel_size_mb[1] ? 256 : 0;
318  } else {
319  ch0size = ctrl->channel_size_mb[0];
320  ch1size = ctrl->channel_size_mb[1];
321  }
322 
323  if (ch0size >= ch1size) {
324  reg = mchbar_read32(MAD_ZR);
325  val = ch1size / 256;
326  reg = (reg & ~0xff000000) | val << 24;
327  reg = (reg & ~0x00ff0000) | (2 * val) << 16;
328  mchbar_write32(MAD_ZR, reg);
329  mchbar_write32(MAD_CHNL, 0x24);
330 
331  } else {
332  reg = mchbar_read32(MAD_ZR);
333  val = ch0size / 256;
334  reg = (reg & ~0xff000000) | val << 24;
335  reg = (reg & ~0x00ff0000) | (2 * val) << 16;
336  mchbar_write32(MAD_ZR, reg);
337  mchbar_write32(MAD_CHNL, 0x21);
338  }
339 }
340 
341 /*
342  * Returns the ECC mode the NB is running at. It takes precedence over ECC capability.
343  * The ME/PCU/.. has the ability to change this.
344  * Return 0: ECC is optional
345  * Return 1: ECC is forced
346  */
347 bool get_host_ecc_forced(void)
348 {
349  /* read Capabilities A Register */
350  const u32 reg32 = pci_read_config32(HOST_BRIDGE, CAPID0_A);
351  return !!(reg32 & (1 << 24));
352 }
353 
354 /*
355  * Returns the ECC capability.
356  * The ME/PCU/.. has the ability to change this.
357  * Return 0: ECC is disabled
358  * Return 1: ECC is possible
359  */
360 bool get_host_ecc_cap(void)
361 {
362  /* read Capabilities A Register */
363  const u32 reg32 = pci_read_config32(HOST_BRIDGE, CAPID0_A);
364  return !(reg32 & (1 << 25));
365 }
366 
367 #define DEFAULT_PCI_MMIO_SIZE 2048
368 
369 void dram_memorymap(ramctr_timing *ctrl, int me_uma_size)
370 {
371  u32 reg, val, reclaim, tom, gfxstolen, gttsize;
372  size_t tsegbase, toludbase, remapbase, gfxstolenbase, mmiosize, gttbase;
373  size_t tsegsize, touudbase, remaplimit, mestolenbase, tsegbasedelta;
374  uint16_t ggc;
375 
376  mmiosize = DEFAULT_PCI_MMIO_SIZE;
377 
378  ggc = pci_read_config16(HOST_BRIDGE, GGC);
379  if (!(ggc & 2)) {
380  gfxstolen = ((ggc >> 3) & 0x1f) * 32;
381  gttsize = ((ggc >> 8) & 0x3);
382  } else {
383  gfxstolen = 0;
384  gttsize = 0;
385  }
386 
387  tsegsize = CONFIG_SMM_TSEG_SIZE >> 20;
388 
389  tom = ctrl->channel_size_mb[0] + ctrl->channel_size_mb[1];
390 
391  mestolenbase = tom - me_uma_size;
392 
393  toludbase = MIN(4096 - mmiosize + gfxstolen + gttsize + tsegsize, tom - me_uma_size);
394 
395  gfxstolenbase = toludbase - gfxstolen;
396  gttbase = gfxstolenbase - gttsize;
397 
398  tsegbase = gttbase - tsegsize;
399 
400  /* Round tsegbase down to nearest address aligned to tsegsize */
401  tsegbasedelta = tsegbase & (tsegsize - 1);
402  tsegbase &= ~(tsegsize - 1);
403 
404  gttbase -= tsegbasedelta;
405  gfxstolenbase -= tsegbasedelta;
406  toludbase -= tsegbasedelta;
407 
408  /* Test if it is possible to reclaim a hole in the RAM addressing */
409  if (tom - me_uma_size > toludbase) {
410  /* Reclaim is possible */
411  reclaim = 1;
412  remapbase = MAX(4096, tom - me_uma_size);
413  remaplimit = remapbase + MIN(4096, tom - me_uma_size) - toludbase - 1;
414  touudbase = remaplimit + 1;
415  } else {
416  /* Reclaim not possible */
417  reclaim = 0;
418  touudbase = tom - me_uma_size;
419  }
420 
421  /* Update memory map in PCIe configuration space */
422  printk(BIOS_DEBUG, "Update PCI-E configuration space:\n");
423 
424  /* TOM (top of memory) */
425  reg = pci_read_config32(HOST_BRIDGE, TOM);
426  val = tom & 0xfff;
427  reg = (reg & ~0xfff00000) | (val << 20);
428  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", TOM, reg);
429  pci_write_config32(HOST_BRIDGE, TOM, reg);
430 
431  reg = pci_read_config32(HOST_BRIDGE, TOM + 4);
432  val = tom & 0xfffff000;
433  reg = (reg & ~0x000fffff) | (val >> 12);
434  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", TOM + 4, reg);
435  pci_write_config32(HOST_BRIDGE, TOM + 4, reg);
436 
437  /* TOLUD (Top Of Low Usable DRAM) */
438  reg = pci_read_config32(HOST_BRIDGE, TOLUD);
439  val = toludbase & 0xfff;
440  reg = (reg & ~0xfff00000) | (val << 20);
441  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", TOLUD, reg);
442  pci_write_config32(HOST_BRIDGE, TOLUD, reg);
443 
444  /* TOUUD LSB (Top Of Upper Usable DRAM) */
445  reg = pci_read_config32(HOST_BRIDGE, TOUUD);
446  val = touudbase & 0xfff;
447  reg = (reg & ~0xfff00000) | (val << 20);
448  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", TOUUD, reg);
449  pci_write_config32(HOST_BRIDGE, TOUUD, reg);
450 
451  /* TOUUD MSB */
452  reg = pci_read_config32(HOST_BRIDGE, TOUUD + 4);
453  val = touudbase & 0xfffff000;
454  reg = (reg & ~0x000fffff) | (val >> 12);
455  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", TOUUD + 4, reg);
456  pci_write_config32(HOST_BRIDGE, TOUUD + 4, reg);
457 
458  if (reclaim) {
459  /* REMAP BASE */
460  pci_write_config32(HOST_BRIDGE, REMAPBASE, remapbase << 20);
461  pci_write_config32(HOST_BRIDGE, REMAPBASE + 4, remapbase >> 12);
462 
463  /* REMAP LIMIT */
464  pci_write_config32(HOST_BRIDGE, REMAPLIMIT, remaplimit << 20);
465  pci_write_config32(HOST_BRIDGE, REMAPLIMIT + 4, remaplimit >> 12);
466  }
467  /* TSEG */
468  reg = pci_read_config32(HOST_BRIDGE, TSEGMB);
469  val = tsegbase & 0xfff;
470  reg = (reg & ~0xfff00000) | (val << 20);
471  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", TSEGMB, reg);
472  pci_write_config32(HOST_BRIDGE, TSEGMB, reg);
473 
474  /* GFX stolen memory */
475  reg = pci_read_config32(HOST_BRIDGE, BDSM);
476  val = gfxstolenbase & 0xfff;
477  reg = (reg & ~0xfff00000) | (val << 20);
478  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", BDSM, reg);
479  pci_write_config32(HOST_BRIDGE, BDSM, reg);
480 
481  /* GTT stolen memory */
482  reg = pci_read_config32(HOST_BRIDGE, BGSM);
483  val = gttbase & 0xfff;
484  reg = (reg & ~0xfff00000) | (val << 20);
485  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", BGSM, reg);
486  pci_write_config32(HOST_BRIDGE, BGSM, reg);
487 
488  if (me_uma_size) {
489  reg = pci_read_config32(HOST_BRIDGE, MESEG_MASK + 4);
490  val = (0x80000 - me_uma_size) & 0xfffff000;
491  reg = (reg & ~0x000fffff) | (val >> 12);
492  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", MESEG_MASK + 4, reg);
493  pci_write_config32(HOST_BRIDGE, MESEG_MASK + 4, reg);
494 
495  /* ME base */
496  reg = pci_read_config32(HOST_BRIDGE, MESEG_BASE);
497  val = mestolenbase & 0xfff;
498  reg = (reg & ~0xfff00000) | (val << 20);
499  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", MESEG_BASE, reg);
500  pci_write_config32(HOST_BRIDGE, MESEG_BASE, reg);
501 
502  reg = pci_read_config32(HOST_BRIDGE, MESEG_BASE + 4);
503  val = mestolenbase & 0xfffff000;
504  reg = (reg & ~0x000fffff) | (val >> 12);
505  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", MESEG_BASE + 4, reg);
506  pci_write_config32(HOST_BRIDGE, MESEG_BASE + 4, reg);
507 
508  /* ME mask */
509  reg = pci_read_config32(HOST_BRIDGE, MESEG_MASK);
510  val = (0x80000 - me_uma_size) & 0xfff;
511  reg = (reg & ~0xfff00000) | (val << 20);
512  reg = reg | ME_STLEN_EN; /* Set ME memory enable */
513  reg = reg | MELCK; /* Set lock bit on ME mem */
514  printk(BIOS_DEBUG, "PCI(0, 0, 0)[%x] = %x\n", MESEG_MASK, reg);
515  pci_write_config32(HOST_BRIDGE, MESEG_MASK, reg);
516  }
517 }
518 
519 static void write_reset(ramctr_timing *ctrl)
520 {
521  int channel, slotrank;
522 
523  /* Choose a populated channel */
524  channel = (ctrl->rankmap[0]) ? 0 : 1;
525 
526  wait_for_iosav(channel);
527 
528  /* Choose a populated rank */
529  slotrank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
530 
531  iosav_write_zqcs_sequence(channel, slotrank, 3, 8, 0);
532 
533  /* This is actually using the IOSAV state machine as a timer */
534  iosav_run_queue(channel, 1, 1);
535 
536  wait_for_iosav(channel);
537 }
538 
539 void dram_jedecreset(ramctr_timing *ctrl)
540 {
541  u32 reg;
542  int channel;
543 
544  while (!(mchbar_read32(RCOMP_TIMER) & (1 << 16)))
545  ;
546  do {
547  reg = mchbar_read32(IOSAV_STATUS_ch(0));
548  } while ((reg & 0x14) == 0);
549 
550  /* Set state of memory controller */
551  reg = 0x112;
552  mchbar_write32(MC_INIT_STATE_G, reg);
553  mchbar_write32(MC_INIT_STATE, 0);
554  reg |= 2; /* DDR reset */
555  mchbar_write32(MC_INIT_STATE_G, reg);
556 
557  /* Assert DIMM reset signal */
558  mchbar_clrbits32(MC_INIT_STATE_G, 1 << 1);
559 
560  /* Wait 200us */
561  udelay(200);
562 
563  /* Deassert DIMM reset signal */
564  mchbar_setbits32(MC_INIT_STATE_G, 1 << 1);
565 
566  /* Wait 500us */
567  udelay(500);
568 
569  /* Enable DCLK */
570  mchbar_setbits32(MC_INIT_STATE_G, 1 << 2);
571 
572  /* XXX Wait 20ns */
573  udelay(1);
574 
575  FOR_ALL_CHANNELS {
576  /* Set valid rank CKE */
577  reg = ctrl->rankmap[channel];
578  mchbar_write32(MC_INIT_STATE_ch(channel), reg);
579 
580  /* Wait 10ns for ranks to settle */
581  // udelay(0.01);
582 
583  reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
584  mchbar_write32(MC_INIT_STATE_ch(channel), reg);
585 
586  /* Write reset using a NOP */
587  write_reset(ctrl);
588  }
589 }
590 
591 /*
592  * DDR3 Rank1 Address mirror swap the following pins:
593  * A3<->A4, A5<->A6, A7<->A8, BA0<->BA1
594  */
595 static void ddr3_mirror_mrreg(int *bank, u32 *addr)
596 {
597  *bank = ((*bank >> 1) & 1) | ((*bank << 1) & 2);
598  *addr = (*addr & ~0x1f8) | ((*addr >> 1) & 0xa8) | ((*addr & 0xa8) << 1);
599 }
600 
601 static void write_mrreg(ramctr_timing *ctrl, int channel, int slotrank, int reg, u32 val)
602 {
603  wait_for_iosav(channel);
604 
605  if (ctrl->rank_mirror[channel][slotrank])
606  ddr3_mirror_mrreg(&reg, &val);
607 
608  const struct iosav_ssq sequence[] = {
609  /* DRAM command MRS */
610  [0] = {
611  .sp_cmd_ctrl = {
612  .command = IOSAV_MRS,
613  },
614  .subseq_ctrl = {
615  .cmd_executions = 1,
616  .cmd_delay_gap = 4,
617  .post_ssq_wait = 4,
618  .data_direction = SSQ_NA,
619  },
620  .sp_cmd_addr = {
621  .address = val,
622  .rowbits = 6,
623  .bank = reg,
624  .rank = slotrank,
625  },
626  },
627  /* DRAM command MRS */
628  [1] = {
629  .sp_cmd_ctrl = {
630  .command = IOSAV_MRS,
631  .ranksel_ap = 1,
632  },
633  .subseq_ctrl = {
634  .cmd_executions = 1,
635  .cmd_delay_gap = 4,
636  .post_ssq_wait = 4,
637  .data_direction = SSQ_NA,
638  },
639  .sp_cmd_addr = {
640  .address = val,
641  .rowbits = 6,
642  .bank = reg,
643  .rank = slotrank,
644  },
645  },
646  /* DRAM command MRS */
647  [2] = {
648  .sp_cmd_ctrl = {
649  .command = IOSAV_MRS,
650  },
651  .subseq_ctrl = {
652  .cmd_executions = 1,
653  .cmd_delay_gap = 4,
654  .post_ssq_wait = ctrl->tMOD,
655  .data_direction = SSQ_NA,
656  },
657  .sp_cmd_addr = {
658  .address = val,
659  .rowbits = 6,
660  .bank = reg,
661  .rank = slotrank,
662  },
663  },
664  };
665  iosav_write_sequence(channel, sequence, ARRAY_SIZE(sequence));
666 
667  iosav_run_once_and_wait(channel);
668 }
669 
670 /* Obtain optimal power down mode for current configuration */
671 static enum power_down_mode get_power_down_mode(ramctr_timing *ctrl)
672 {
673  if (ctrl->tXP > 8)
674  return PDM_NONE;
675 
676  if (ctrl->tXPDLL > 32)
677  return PDM_PPD;
678 
679  if (CONFIG(RAMINIT_ALWAYS_ALLOW_DLL_OFF) || get_platform_type() == PLATFORM_MOBILE)
680  return PDM_DLL_OFF;
681 
682  return PDM_APD_PPD;
683 }
684 
685 static u32 make_mr0(ramctr_timing *ctrl, u8 rank)
686 {
687  u16 mr0reg, mch_cas, mch_wr;
688  static const u8 mch_wr_t[12] = { 1, 2, 3, 4, 0, 5, 0, 6, 0, 7, 0, 0 };
689 
690  const enum power_down_mode power_down = get_power_down_mode(ctrl);
691 
692  const bool slow_exit = power_down == PDM_DLL_OFF || power_down == PDM_APD_DLL_OFF;
693 
694  /* Convert CAS to MCH register friendly */
695  if (ctrl->CAS < 12) {
696  mch_cas = (u16) ((ctrl->CAS - 4) << 1);
697  } else {
698  mch_cas = (u16) (ctrl->CAS - 12);
699  mch_cas = ((mch_cas << 1) | 0x1);
700  }
701 
702  /* Convert tWR to MCH register friendly */
703  mch_wr = mch_wr_t[ctrl->tWR - 5];
704 
705  /* DLL Reset - self clearing - set after CLK frequency has been changed */
706  mr0reg = 1 << 8;
707 
708  mr0reg |= (mch_cas & 0x1) << 2;
709  mr0reg |= (mch_cas & 0xe) << 3;
710  mr0reg |= mch_wr << 9;
711 
712  /* Precharge PD - Use slow exit when DLL-off is used - mostly power-saving feature */
713  mr0reg |= !slow_exit << 12;
714  return mr0reg;
715 }
716 
717 static void dram_mr0(ramctr_timing *ctrl, u8 rank, int channel)
718 {
719  write_mrreg(ctrl, channel, rank, 0, make_mr0(ctrl, rank));
720 }
721 
722 static odtmap get_ODT(ramctr_timing *ctrl, int channel)
723 {
724  /* Get ODT based on rankmap */
725  int dimms_per_ch = (ctrl->rankmap[channel] & 1) + ((ctrl->rankmap[channel] >> 2) & 1);
726 
727  if (dimms_per_ch == 1) {
728  return (const odtmap){60, 60};
729  } else {
730  return (const odtmap){120, 30};
731  }
732 }
733 
734 static u32 encode_odt(u32 odt)
735 {
736  switch (odt) {
737  case 30:
738  return (1 << 9) | (1 << 2); /* RZQ/8, RZQ/4 */
739  case 60:
740  return (1 << 2); /* RZQ/4 */
741  case 120:
742  return (1 << 6); /* RZQ/2 */
743  default:
744  case 0:
745  return 0;
746  }
747 }
748 
749 static u32 make_mr1(ramctr_timing *ctrl, u8 rank, int channel)
750 {
751  odtmap odt;
752  u32 mr1reg;
753 
754  odt = get_ODT(ctrl, channel);
755  mr1reg = 2;
756 
757  mr1reg |= encode_odt(odt.rttnom);
758 
759  return mr1reg;
760 }
761 
762 static void dram_mr1(ramctr_timing *ctrl, u8 rank, int channel)
763 {
764  u16 mr1reg;
765 
766  mr1reg = make_mr1(ctrl, rank, channel);
767 
768  write_mrreg(ctrl, channel, rank, 1, mr1reg);
769 }
770 
771 static void dram_mr2(ramctr_timing *ctrl, u8 rank, int channel)
772 {
773  const u16 pasr = 0;
774  const u16 cwl = ctrl->CWL - 5;
775  const odtmap odt = get_ODT(ctrl, channel);
776 
777  int srt = 0;
778  if (IS_IVY_CPU(ctrl->cpu) && ctrl->tCK >= TCK_1066MHZ)
779  srt = ctrl->extended_temperature_range && !ctrl->auto_self_refresh;
780 
781  u16 mr2reg = 0;
782  mr2reg |= pasr;
783  mr2reg |= cwl << 3;
784  mr2reg |= ctrl->auto_self_refresh << 6;
785  mr2reg |= srt << 7;
786  mr2reg |= (odt.rttwr / 60) << 9;
787 
788  write_mrreg(ctrl, channel, rank, 2, mr2reg);
789 
790  /* Program MR2 shadow */
791  u32 reg32 = mchbar_read32(TC_MR2_SHADOW_ch(channel));
792 
793  reg32 &= 3 << 14 | 3 << 6;
794 
795  reg32 |= mr2reg & ~(3 << 6);
796 
797  if (srt)
798  reg32 |= 1 << (rank / 2 + 6);
799 
800  if (ctrl->rank_mirror[channel][rank])
801  reg32 |= 1 << (rank / 2 + 14);
802 
803  mchbar_write32(TC_MR2_SHADOW_ch(channel), reg32);
804 }
805 
806 static void dram_mr3(ramctr_timing *ctrl, u8 rank, int channel)
807 {
808  write_mrreg(ctrl, channel, rank, 3, 0);
809 }
810 
811 void dram_mrscommands(ramctr_timing *ctrl)
812 {
813  u8 slotrank;
814  int channel;
815 
816  FOR_ALL_POPULATED_CHANNELS {
817  FOR_ALL_POPULATED_RANKS {
818  /* MR2 */
819  dram_mr2(ctrl, slotrank, channel);
820 
821  /* MR3 */
822  dram_mr3(ctrl, slotrank, channel);
823 
824  /* MR1 */
825  dram_mr1(ctrl, slotrank, channel);
826 
827  /* MR0 */
828  dram_mr0(ctrl, slotrank, channel);
829  }
830  }
831 
832  const struct iosav_ssq zqcl_sequence[] = {
833  /* DRAM command NOP (without ODT nor chip selects) */
834  [0] = {
835  .sp_cmd_ctrl = {
836  .command = IOSAV_NOP & ~(0xff << 8),
837  },
838  .subseq_ctrl = {
839  .cmd_executions = 1,
840  .cmd_delay_gap = 4,
841  .post_ssq_wait = 15,
842  .data_direction = SSQ_NA,
843  },
844  .sp_cmd_addr = {
845  .address = 2,
846  .rowbits = 6,
847  .bank = 0,
848  .rank = 0,
849  },
850  },
851  /* DRAM command ZQCL */
852  [1] = {
853  .sp_cmd_ctrl = {
854  .command = IOSAV_ZQCS,
855  .ranksel_ap = 1,
856  },
857  .subseq_ctrl = {
858  .cmd_executions = 1,
859  .cmd_delay_gap = 4,
860  .post_ssq_wait = 400,
861  .data_direction = SSQ_NA,
862  },
863  .sp_cmd_addr = {
864  .address = 1 << 10,
865  .rowbits = 6,
866  .bank = 0,
867  .rank = 0,
868  },
869  .addr_update = {
870  .inc_rank = 1,
871  .addr_wrap = 20,
872  },
873  },
874  };
875  iosav_write_sequence(BROADCAST_CH, zqcl_sequence, ARRAY_SIZE(zqcl_sequence));
876 
877  iosav_run_queue(BROADCAST_CH, 4, 0);
878 
879  FOR_ALL_CHANNELS {
880  wait_for_iosav(channel);
881  }
882 
883  /* Refresh enable */
884  mchbar_setbits32(MC_INIT_STATE_G, 1 << 3);
885 
886  FOR_ALL_POPULATED_CHANNELS {
887  mchbar_clrbits32(SCHED_CBIT_ch(channel), 1 << 21);
888 
889  wait_for_iosav(channel);
890 
891  slotrank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
892 
893  wait_for_iosav(channel);
894 
895  iosav_write_zqcs_sequence(channel, slotrank, 4, 101, 31);
896 
897  iosav_run_once_and_wait(channel);
898  }
899 }
900 
901 static const u32 lane_base[] = {
902  LANEBASE_B0, LANEBASE_B1, LANEBASE_B2, LANEBASE_B3,
903  LANEBASE_B4, LANEBASE_B5, LANEBASE_B6, LANEBASE_B7,
904  LANEBASE_ECC
905 };
906 
907 /* Maximum delay for command, control, clock */
908 #define CCC_MAX_PI (2 * QCLK_PI - 1)
909 
910 void program_timings(ramctr_timing *ctrl, int channel)
911 {
912  u32 reg_roundtrip_latency, reg_io_latency;
913  int lane;
914  int slotrank, slot;
915 
916  u32 ctl_delay[NUM_SLOTS] = { 0 };
917  int cmd_delay = 0;
918 
919  /* Enable CLK XOVER */
920  u32 clk_pi_coding = get_XOVER_CLK(ctrl->rankmap[channel]);
921  u32 clk_logic_dly = 0;
922 
923  /*
924  * Compute command timing as abs() of the most negative PI code
925  * across all ranks. Use zero if none of the values is negative.
926  */
927  FOR_ALL_POPULATED_RANKS {
928  cmd_delay = MAX(cmd_delay, -ctrl->timings[channel][slotrank].pi_coding);
929  }
930  if (cmd_delay > CCC_MAX_PI) {
931  printk(BIOS_ERR, "C%d command delay overflow: %d\n", channel, cmd_delay);
932  cmd_delay = CCC_MAX_PI;
933  }
934 
935  for (slot = 0; slot < NUM_SLOTS; slot++) {
936  const int pi_coding_0 = ctrl->timings[channel][2 * slot + 0].pi_coding;
937  const int pi_coding_1 = ctrl->timings[channel][2 * slot + 1].pi_coding;
938 
939  const u8 slot_map = (ctrl->rankmap[channel] >> (2 * slot)) & 3;
940 
941  if (slot_map & 1)
942  ctl_delay[slot] += pi_coding_0 + cmd_delay;
943 
944  if (slot_map & 2)
945  ctl_delay[slot] += pi_coding_1 + cmd_delay;
946 
947  /* If both ranks in a slot are populated, use the average */
948  if (slot_map == 3)
949  ctl_delay[slot] /= 2;
950 
951  if (ctl_delay[slot] > CCC_MAX_PI) {
952  printk(BIOS_ERR, "C%dS%d control delay overflow: %d\n",
953  channel, slot, ctl_delay[slot]);
954  ctl_delay[slot] = CCC_MAX_PI;
955  }
956  }
957  FOR_ALL_POPULATED_RANKS {
958  int clk_delay = ctrl->timings[channel][slotrank].pi_coding + cmd_delay;
959 
960  /*
961  * Clock is a differential signal, whereas command and control are not.
962  * This affects its timing, and it is also why it needs a magic offset.
963  */
964  clk_delay += ctrl->pi_code_offset;
965 
966  /* Can never happen with valid values */
967  if (clk_delay < 0) {
968  printk(BIOS_ERR, "C%dR%d clock delay underflow: %d\n",
969  channel, slotrank, clk_delay);
970  clk_delay = 0;
971  }
972 
973  /* Clock can safely wrap around because it is a periodic signal */
974  clk_delay %= CCC_MAX_PI + 1;
975 
976  clk_pi_coding |= (clk_delay % QCLK_PI) << (6 * slotrank);
977  clk_logic_dly |= (clk_delay / QCLK_PI) << slotrank;
978  }
979 
980  /* Enable CMD XOVER */
981  union gdcr_cmd_pi_coding_reg cmd_pi_coding = {
982  .raw = get_XOVER_CMD(ctrl->rankmap[channel]),
983  };
984  cmd_pi_coding.cmd_pi_code = cmd_delay % QCLK_PI;
985  cmd_pi_coding.cmd_logic_delay = cmd_delay / QCLK_PI;
986 
987  cmd_pi_coding.ctl_pi_code_d0 = ctl_delay[0] % QCLK_PI;
988  cmd_pi_coding.ctl_pi_code_d1 = ctl_delay[1] % QCLK_PI;
989  cmd_pi_coding.ctl_logic_delay_d0 = ctl_delay[0] / QCLK_PI;
990  cmd_pi_coding.ctl_logic_delay_d1 = ctl_delay[1] / QCLK_PI;
991 
992  mchbar_write32(GDCRCMDPICODING_ch(channel), cmd_pi_coding.raw);
993 
994  mchbar_write32(GDCRCKPICODE_ch(channel), clk_pi_coding);
995  mchbar_write32(GDCRCKLOGICDELAY_ch(channel), clk_logic_dly);
996 
997  reg_io_latency = mchbar_read32(SC_IO_LATENCY_ch(channel));
998  reg_io_latency &= ~0xffff;
999 
1000  reg_roundtrip_latency = 0;
1001 
1002  FOR_ALL_POPULATED_RANKS {
1003  reg_io_latency |= ctrl->timings[channel][slotrank].io_latency << (4 * slotrank);
1004 
1005  reg_roundtrip_latency |=
1006  ctrl->timings[channel][slotrank].roundtrip_latency << (8 * slotrank);
1007 
1008  FOR_ALL_LANES {
1009  const u16 rcven = ctrl->timings[channel][slotrank].lanes[lane].rcven;
1010  const u8 dqs_p = ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_p;
1011  const u8 dqs_n = ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_n;
1012  const union gdcr_rx_reg gdcr_rx = {
1013  .rcven_pi_code = rcven % QCLK_PI,
1014  .rx_dqs_p_pi_code = dqs_p,
1015  .rcven_logic_delay = rcven / QCLK_PI,
1016  .rx_dqs_n_pi_code = dqs_n,
1017  };
1018  mchbar_write32(lane_base[lane] + GDCRRX(channel, slotrank),
1019  gdcr_rx.raw);
1020 
1021  const u16 tx_dqs = ctrl->timings[channel][slotrank].lanes[lane].tx_dqs;
1022  const int tx_dq = ctrl->timings[channel][slotrank].lanes[lane].tx_dq;
1023  const union gdcr_tx_reg gdcr_tx = {
1024  .tx_dq_pi_code = tx_dq % QCLK_PI,
1025  .tx_dqs_pi_code = tx_dqs % QCLK_PI,
1026  .tx_dqs_logic_delay = tx_dqs / QCLK_PI,
1027  .tx_dq_logic_delay = tx_dq / QCLK_PI,
1028  };
1029  mchbar_write32(lane_base[lane] + GDCRTX(channel, slotrank),
1030  gdcr_tx.raw);
1031  }
1032  }
1033  mchbar_write32(SC_ROUNDT_LAT_ch(channel), reg_roundtrip_latency);
1034  mchbar_write32(SC_IO_LATENCY_ch(channel), reg_io_latency);
1035 }
1036 
1037 static void test_rcven(ramctr_timing *ctrl, int channel, int slotrank)
1038 {
1039  wait_for_iosav(channel);
1040 
1041  /* Send a burst of 16 back-to-back read commands (4 DCLK apart) */
1042  iosav_write_read_mpr_sequence(channel, slotrank, ctrl->tMOD, 1, 3, 15, ctrl->CAS + 36);
1043 
1044  iosav_run_once_and_wait(channel);
1045 }
1046 
1047 static int does_lane_work(ramctr_timing *ctrl, int channel, int slotrank, int lane)
1048 {
1049  u32 rcven = ctrl->timings[channel][slotrank].lanes[lane].rcven;
1050 
1051  return (mchbar_read32(lane_base[lane] +
1052  GDCRTRAININGRESULT(channel, (rcven / 32) & 1)) >> (rcven % 32)) & 1;
1053 }
1054 
1055 struct run {
1056  int middle;
1057  int end;
1058  int start;
1059  int all;
1060  int length;
1061 };
1062 
1063 static struct run get_longest_zero_run(int *seq, int sz)
1064 {
1065  int i, ls;
1066  int bl = 0, bs = 0;
1067  struct run ret;
1068 
1069  ls = 0;
1070  for (i = 0; i < 2 * sz; i++)
1071  if (seq[i % sz]) {
1072  if (i - ls > bl) {
1073  bl = i - ls;
1074  bs = ls;
1075  }
1076  ls = i + 1;
1077  }
1078  if (bl == 0) {
1079  ret.middle = sz / 2;
1080  ret.start = 0;
1081  ret.end = sz;
1082  ret.length = sz;
1083  ret.all = 1;
1084  return ret;
1085  }
1086 
1087  ret.start = bs % sz;
1088  ret.end = (bs + bl - 1) % sz;
1089  ret.middle = (bs + (bl - 1) / 2) % sz;
1090  ret.length = bl;
1091  ret.all = 0;
1092 
1093  return ret;
1094 }
1095 
1096 #define RCVEN_COARSE_PI_LENGTH (2 * QCLK_PI)
1097 
1098 static void find_rcven_pi_coarse(ramctr_timing *ctrl, int channel, int slotrank, int *upperA)
1099 {
1100  int rcven;
1101  int statistics[NUM_LANES][RCVEN_COARSE_PI_LENGTH];
1102  int lane;
1103 
1104  for (rcven = 0; rcven < RCVEN_COARSE_PI_LENGTH; rcven++) {
1105  FOR_ALL_LANES {
1106  ctrl->timings[channel][slotrank].lanes[lane].rcven = rcven;
1107  }
1108  program_timings(ctrl, channel);
1109 
1110  test_rcven(ctrl, channel, slotrank);
1111 
1112  FOR_ALL_LANES {
1113  statistics[lane][rcven] =
1114  !does_lane_work(ctrl, channel, slotrank, lane);
1115  }
1116  }
1117  FOR_ALL_LANES {
1118  struct run rn = get_longest_zero_run(statistics[lane], RCVEN_COARSE_PI_LENGTH);
1119  ctrl->timings[channel][slotrank].lanes[lane].rcven = rn.middle;
1120  upperA[lane] = rn.end;
1121  if (upperA[lane] < rn.middle)
1122  upperA[lane] += 2 * QCLK_PI;
1123 
1124  printram("rcven: %d, %d, %d: % 4d-% 4d-% 4d\n",
1125  channel, slotrank, lane, rn.start, rn.middle, rn.end);
1126  }
1127 }
1128 
1129 static void fine_tune_rcven_pi(ramctr_timing *ctrl, int channel, int slotrank, int *upperA)
1130 {
1131  int rcven_delta;
1132  int statistics[NUM_LANES][51] = {0};
1133  int lane, i;
1134 
1135  for (rcven_delta = -25; rcven_delta <= 25; rcven_delta++) {
1136 
1137  FOR_ALL_LANES {
1138  ctrl->timings[channel][slotrank].lanes[lane].rcven
1139  = upperA[lane] + rcven_delta + QCLK_PI;
1140  }
1141  program_timings(ctrl, channel);
1142 
1143  for (i = 0; i < 100; i++) {
1144  test_rcven(ctrl, channel, slotrank);
1145  FOR_ALL_LANES {
1146  statistics[lane][rcven_delta + 25] +=
1147  does_lane_work(ctrl, channel, slotrank, lane);
1148  }
1149  }
1150  }
1151  FOR_ALL_LANES {
1152  int last_zero, first_all;
1153 
1154  for (last_zero = -25; last_zero <= 25; last_zero++)
1155  if (statistics[lane][last_zero + 25])
1156  break;
1157 
1158  last_zero--;
1159  for (first_all = -25; first_all <= 25; first_all++)
1160  if (statistics[lane][first_all + 25] == 100)
1161  break;
1162 
1163  printram("lane %d: %d, %d\n", lane, last_zero, first_all);
1164 
1165  ctrl->timings[channel][slotrank].lanes[lane].rcven =
1166  (last_zero + first_all) / 2 + upperA[lane];
1167 
1168  printram("Aval: %d, %d, %d: % 4d\n", channel, slotrank,
1169  lane, ctrl->timings[channel][slotrank].lanes[lane].rcven);
1170  }
1171 }
1172 
1173 /*
1174  * Once the DQS high phase has been found (for each DRAM) the next stage
1175  * is to find out the round trip latency, by locating the preamble cycle.
1176  * This is achieved by trying smaller and smaller roundtrip values until
1177  * the strobe sampling is done on the preamble cycle.
1178  */
1179 static int find_roundtrip_latency(ramctr_timing *ctrl, int channel, int slotrank, int *upperA)
1180 {
1181  int works[NUM_LANES];
1182  int lane;
1183 
1184  while (1) {
1185  int all_works = 1, some_works = 0;
1186 
1187  program_timings(ctrl, channel);
1188  test_rcven(ctrl, channel, slotrank);
1189 
1190  FOR_ALL_LANES {
1191  works[lane] = !does_lane_work(ctrl, channel, slotrank, lane);
1192 
1193  if (works[lane])
1194  some_works = 1;
1195  else
1196  all_works = 0;
1197  }
1198 
1199  /* If every lane is working, exit */
1200  if (all_works)
1201  return 0;
1202 
1203  /*
1204  * If all bits are one (everyone is failing), decrement
1205  * the roundtrip value by two, and do another iteration.
1206  */
1207  if (!some_works) {
1208  /* Guard against roundtrip latency underflow */
1209  if (ctrl->timings[channel][slotrank].roundtrip_latency < 2) {
1210  printk(BIOS_EMERG, "Roundtrip latency underflow: %d, %d\n",
1211  channel, slotrank);
1212  return MAKE_ERR;
1213  }
1214  ctrl->timings[channel][slotrank].roundtrip_latency -= 2;
1215  printram("4024 -= 2;\n");
1216  continue;
1217  }
1218 
1219  /*
1220  * Else (if some lanes are failing), increase the rank's
1221  * I/O latency by 2, and increase rcven logic delay by 2
1222  * on the working lanes, then perform another iteration.
1223  */
1224  ctrl->timings[channel][slotrank].io_latency += 2;
1225  printram("4028 += 2;\n");
1226 
1227  /* Guard against I/O latency overflow */
1228  if (ctrl->timings[channel][slotrank].io_latency >= 16) {
1229  printk(BIOS_EMERG, "I/O latency overflow: %d, %d\n",
1230  channel, slotrank);
1231  return MAKE_ERR;
1232  }
1233  FOR_ALL_LANES if (works[lane]) {
1234  ctrl->timings[channel][slotrank].lanes[lane].rcven += 2 * QCLK_PI;
1235  upperA[lane] += 2 * QCLK_PI;
1236  printram("increment %d, %d, %d\n", channel, slotrank, lane);
1237  }
1238  }
1239  return 0;
1240 }
1241 
1242 static int get_logic_delay_delta(ramctr_timing *ctrl, int channel, int slotrank)
1243 {
1244  int lane;
1245  u16 logic_delay_min = 7;
1246  u16 logic_delay_max = 0;
1247 
1248  FOR_ALL_LANES {
1249  const u16 logic_delay = ctrl->timings[channel][slotrank].lanes[lane].rcven >> 6;
1250 
1251  logic_delay_min = MIN(logic_delay_min, logic_delay);
1252  logic_delay_max = MAX(logic_delay_max, logic_delay);
1253  }
1254 
1255  if (logic_delay_max < logic_delay_min) {
1256  printk(BIOS_EMERG, "Logic delay max < min (%u < %u): %d, %d\n",
1257  logic_delay_max, logic_delay_min, channel, slotrank);
1258  }
1259 
1260  assert(logic_delay_max >= logic_delay_min);
1261 
1262  return logic_delay_max - logic_delay_min;
1263 }
1264 
1265 static int align_rt_io_latency(ramctr_timing *ctrl, int channel, int slotrank, int prev)
1266 {
1267  int latency_offset = 0;
1268 
1269  /* Get changed maxima */
1270  const int post = get_logic_delay_delta(ctrl, channel, slotrank);
1271 
1272  if (prev < post)
1273  latency_offset = +1;
1274 
1275  else if (prev > post)
1276  latency_offset = -1;
1277 
1278  else
1279  latency_offset = 0;
1280 
1281  ctrl->timings[channel][slotrank].io_latency += latency_offset;
1282  ctrl->timings[channel][slotrank].roundtrip_latency += latency_offset;
1283  printram("4024 += %d;\n", latency_offset);
1284  printram("4028 += %d;\n", latency_offset);
1285 
1286  return post;
1287 }
1288 
1289 static void compute_final_logic_delay(ramctr_timing *ctrl, int channel, int slotrank)
1290 {
1291  u16 logic_delay_min = 7;
1292  int lane;
1293 
1294  FOR_ALL_LANES {
1295  const u16 logic_delay = ctrl->timings[channel][slotrank].lanes[lane].rcven >> 6;
1296 
1297  logic_delay_min = MIN(logic_delay_min, logic_delay);
1298  }
1299 
1300  if (logic_delay_min >= 2) {
1301  printk(BIOS_WARNING, "Logic delay %u greater than 1: %d %d\n",
1302  logic_delay_min, channel, slotrank);
1303  }
1304 
1305  FOR_ALL_LANES {
1306  ctrl->timings[channel][slotrank].lanes[lane].rcven -= logic_delay_min << 6;
1307  }
1308  ctrl->timings[channel][slotrank].io_latency -= logic_delay_min;
1309  printram("4028 -= %d;\n", logic_delay_min);
1310 }
1311 
1312 int receive_enable_calibration(ramctr_timing *ctrl)
1313 {
1314  int channel, slotrank, lane;
1315  int err;
1316 
1317  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
1318  int all_high, some_high;
1319  int upperA[NUM_LANES];
1320  int prev;
1321 
1322  wait_for_iosav(channel);
1323 
1324  iosav_write_prea_sequence(channel, slotrank, ctrl->tRP, 0);
1325 
1326  iosav_run_once_and_wait(channel);
1327 
1328  const union gdcr_training_mod_reg training_mod = {
1329  .receive_enable_mode = 1,
1330  .training_rank_sel = slotrank,
1331  .odt_always_on = 1,
1332  };
1333  mchbar_write32(GDCRTRAININGMOD, training_mod.raw);
1334 
1335  ctrl->timings[channel][slotrank].io_latency = 4;
1336  ctrl->timings[channel][slotrank].roundtrip_latency = 55;
1337  program_timings(ctrl, channel);
1338 
1339  find_rcven_pi_coarse(ctrl, channel, slotrank, upperA);
1340 
1341  all_high = 1;
1342  some_high = 0;
1343  FOR_ALL_LANES {
1344  if (ctrl->timings[channel][slotrank].lanes[lane].rcven >= QCLK_PI)
1345  some_high = 1;
1346  else
1347  all_high = 0;
1348  }
1349 
1350  if (all_high) {
1351  ctrl->timings[channel][slotrank].io_latency--;
1352  printram("4028--;\n");
1353  FOR_ALL_LANES {
1354  ctrl->timings[channel][slotrank].lanes[lane].rcven -= QCLK_PI;
1355  upperA[lane] -= QCLK_PI;
1356 
1357  }
1358  } else if (some_high) {
1359  ctrl->timings[channel][slotrank].roundtrip_latency++;
1360  ctrl->timings[channel][slotrank].io_latency++;
1361  printram("4024++;\n");
1362  printram("4028++;\n");
1363  }
1364 
1365  program_timings(ctrl, channel);
1366 
1367  prev = get_logic_delay_delta(ctrl, channel, slotrank);
1368 
1369  err = find_roundtrip_latency(ctrl, channel, slotrank, upperA);
1370  if (err)
1371  return err;
1372 
1373  prev = align_rt_io_latency(ctrl, channel, slotrank, prev);
1374 
1375  fine_tune_rcven_pi(ctrl, channel, slotrank, upperA);
1376 
1377  prev = align_rt_io_latency(ctrl, channel, slotrank, prev);
1378 
1379  compute_final_logic_delay(ctrl, channel, slotrank);
1380 
1381  align_rt_io_latency(ctrl, channel, slotrank, prev);
1382 
1383  printram("4/8: %d, %d, % 4d, % 4d\n", channel, slotrank,
1384  ctrl->timings[channel][slotrank].roundtrip_latency,
1385  ctrl->timings[channel][slotrank].io_latency);
1386 
1387  printram("final results:\n");
1388  FOR_ALL_LANES
1389  printram("Aval: %d, %d, %d: % 4d\n", channel, slotrank, lane,
1390  ctrl->timings[channel][slotrank].lanes[lane].rcven);
1391 
1392  mchbar_write32(GDCRTRAININGMOD, 0);
1393 
1394  toggle_io_reset();
1395  }
1396 
1397  FOR_ALL_POPULATED_CHANNELS {
1398  program_timings(ctrl, channel);
1399  }
1400 
1401  return 0;
1402 }
1403 
1404 static void test_tx_dq(ramctr_timing *ctrl, int channel, int slotrank)
1405 {
1406  int lane;
1407 
1408  FOR_ALL_LANES {
1409  mchbar_write32(IOSAV_By_ERROR_COUNT_ch(channel, lane), 0);
1410  mchbar_read32(IOSAV_By_BW_SERROR_C_ch(channel, lane));
1411  }
1412 
1413  wait_for_iosav(channel);
1414 
1415  iosav_write_misc_write_sequence(ctrl, channel, slotrank,
1416  MAX(ctrl->tRRD, (ctrl->tFAW >> 2) + 1), 4, 4, 500, 18);
1417 
1418  iosav_run_once_and_wait(channel);
1419 
1420  iosav_write_prea_act_read_sequence(ctrl, channel, slotrank);
1421 
1422  iosav_run_once_and_wait(channel);
1423 }
1424 
1425 static void tx_dq_threshold_process(int *data, const int count)
1426 {
1427  int min = data[0];
1428  int max = min;
1429  int i;
1430  for (i = 1; i < count; i++) {
1431  if (min > data[i])
1432  min = data[i];
1433 
1434  if (max < data[i])
1435  max = data[i];
1436  }
1437  int threshold = min / 2 + max / 2;
1438  for (i = 0; i < count; i++)
1439  data[i] = data[i] > threshold;
1440 
1441  printram("threshold=%d min=%d max=%d\n", threshold, min, max);
1442 }
1443 
1444 static int tx_dq_write_leveling(ramctr_timing *ctrl, int channel, int slotrank)
1445 {
1446  int tx_dq;
1447  int stats[NUM_LANES][MAX_TX_DQ + 1];
1448  int lane;
1449 
1450  wait_for_iosav(channel);
1451 
1452  iosav_write_prea_sequence(channel, slotrank, ctrl->tRP, 18);
1453 
1454  iosav_run_once_and_wait(channel);
1455 
1456  for (tx_dq = 0; tx_dq <= MAX_TX_DQ; tx_dq++) {
1457  FOR_ALL_LANES ctrl->timings[channel][slotrank].lanes[lane].tx_dq = tx_dq;
1458  program_timings(ctrl, channel);
1459 
1460  test_tx_dq(ctrl, channel, slotrank);
1461 
1462  FOR_ALL_LANES {
1463  stats[lane][tx_dq] = mchbar_read32(
1464  IOSAV_By_ERROR_COUNT_ch(channel, lane));
1465  }
1466  }
1467  FOR_ALL_LANES {
1468  struct run rn = get_longest_zero_run(stats[lane], ARRAY_SIZE(stats[lane]));
1469 
1470  if (rn.all || rn.length < 8) {
1471  printk(BIOS_EMERG, "tx_dq write leveling failed: %d, %d, %d\n",
1472  channel, slotrank, lane);
1473  /*
1474  * With command training not being done yet, the lane can be erroneous.
1475  * Take the average as reference and try again to find a run.
1476  */
1477  tx_dq_threshold_process(stats[lane], ARRAY_SIZE(stats[lane]));
1478  rn = get_longest_zero_run(stats[lane], ARRAY_SIZE(stats[lane]));
1479 
1480  if (rn.all || rn.length < 8) {
1481  printk(BIOS_EMERG, "tx_dq recovery failed\n");
1482  return MAKE_ERR;
1483  }
1484  }
1485  ctrl->timings[channel][slotrank].lanes[lane].tx_dq = rn.middle;
1486  printram("tx_dq: %d, %d, %d: % 4d-% 4d-% 4d\n",
1487  channel, slotrank, lane, rn.start, rn.middle, rn.end);
1488  }
1489  return 0;
1490 }
1491 
1492 static int get_precedening_channels(ramctr_timing *ctrl, int target_channel)
1493 {
1494  int channel, ret = 0;
1495 
1496  FOR_ALL_POPULATED_CHANNELS if (channel < target_channel)
1497  ret++;
1498 
1499  return ret;
1500 }
1501 
1502 /* Each cacheline is 64 bits long */
1503 static void program_wdb_pattern_length(int channel, const unsigned int num_cachelines)
1504 {
1505  mchbar_write8(IOSAV_DATA_CTL_ch(channel), num_cachelines / 8 - 1);
1506 }
1507 
1508 static void fill_pattern0(ramctr_timing *ctrl, int channel, u32 a, u32 b)
1509 {
1510  unsigned int j;
1511  unsigned int channel_offset = get_precedening_channels(ctrl, channel) * 64;
1512  uintptr_t addr;
1513 
1514  for (j = 0; j < 16; j++) {
1515  addr = 0x04000000 + channel_offset + 4 * j;
1516  write32((void *)addr, j & 2 ? b : a);
1517  }
1518 
1519  sfence();
1520 
1521  program_wdb_pattern_length(channel, 8);
1522 }
1523 
1524 static int num_of_channels(const ramctr_timing *ctrl)
1525 {
1526  int ret = 0;
1527  int channel;
1528  FOR_ALL_POPULATED_CHANNELS ret++;
1529  return ret;
1530 }
1531 
1532 static void fill_pattern1(ramctr_timing *ctrl, int channel)
1533 {
1534  unsigned int j;
1535  unsigned int channel_offset = get_precedening_channels(ctrl, channel) * 64;
1536  unsigned int channel_step = 64 * num_of_channels(ctrl);
1537  uintptr_t addr;
1538 
1539  for (j = 0; j < 16; j++) {
1540  addr = 0x04000000 + channel_offset + j * 4;
1541  write32((void *)addr, 0xffffffff);
1542  }
1543  for (j = 0; j < 16; j++) {
1544  addr = 0x04000000 + channel_offset + channel_step + j * 4;
1545  write32((void *)addr, 0);
1546  }
1547  sfence();
1548 
1549  program_wdb_pattern_length(channel, 16);
1550 }
1551 
1552 #define TX_DQS_PI_LENGTH (2 * QCLK_PI)
1553 
1554 static int write_level_rank(ramctr_timing *ctrl, int channel, int slotrank)
1555 {
1556  int tx_dqs;
1557  int statistics[NUM_LANES][TX_DQS_PI_LENGTH];
1558  int lane;
1559 
1560  const union gdcr_training_mod_reg training_mod = {
1561  .write_leveling_mode = 1,
1562  .training_rank_sel = slotrank,
1563  .enable_dqs_wl = 5,
1564  .odt_always_on = 1,
1565  .force_drive_enable = 1,
1566  };
1567  mchbar_write32(GDCRTRAININGMOD, training_mod.raw);
1568 
1569  u32 mr1reg = make_mr1(ctrl, slotrank, channel) | 1 << 7;
1570  int bank = 1;
1571 
1572  if (ctrl->rank_mirror[channel][slotrank])
1573  ddr3_mirror_mrreg(&bank, &mr1reg);
1574 
1575  wait_for_iosav(channel);
1576 
1577  iosav_write_jedec_write_leveling_sequence(ctrl, channel, slotrank, bank, mr1reg);
1578 
1579  for (tx_dqs = 0; tx_dqs < TX_DQS_PI_LENGTH; tx_dqs++) {
1580  FOR_ALL_LANES {
1581  ctrl->timings[channel][slotrank].lanes[lane].tx_dqs = tx_dqs;
1582  }
1583  program_timings(ctrl, channel);
1584 
1585  iosav_run_once_and_wait(channel);
1586 
1587  FOR_ALL_LANES {
1588  statistics[lane][tx_dqs] = !((mchbar_read32(lane_base[lane] +
1589  GDCRTRAININGRESULT(channel, (tx_dqs / 32) & 1)) >>
1590  (tx_dqs % 32)) & 1);
1591  }
1592  }
1593  FOR_ALL_LANES {
1594  struct run rn = get_longest_zero_run(statistics[lane], TX_DQS_PI_LENGTH);
1595  /*
1596  * tx_dq is a direct function of tx_dqs's 6 LSBs. Some tests increment the value
1597  * of tx_dqs by a small value, which might cause the 6-bit value to overflow if
1598  * it's close to 0x3f. Increment the value by a small offset if it's likely
1599  * to overflow, to make sure it won't overflow while running tests and bricks
1600  * the system due to a non matching tx_dq.
1601  *
1602  * TODO: find out why some tests (edge write discovery) increment tx_dqs.
1603  */
1604  if ((rn.start & 0x3f) == 0x3e)
1605  rn.start += 2;
1606  else if ((rn.start & 0x3f) == 0x3f)
1607  rn.start += 1;
1608 
1609  ctrl->timings[channel][slotrank].lanes[lane].tx_dqs = rn.start;
1610  if (rn.all) {
1611  printk(BIOS_EMERG, "JEDEC write leveling failed: %d, %d, %d\n",
1612  channel, slotrank, lane);
1613 
1614  return MAKE_ERR;
1615  }
1616  printram("tx_dqs: %d, %d, %d: % 4d-% 4d-% 4d\n",
1617  channel, slotrank, lane, rn.start, rn.middle, rn.end);
1618  }
1619  return 0;
1620 }
1621 
1622 static int get_dqs_flyby_adjust(u64 val)
1623 {
1624  int i;
1625  /* DQS is good enough */
1626  if (val == 0xffffffffffffffffLL)
1627  return 0;
1628  if (val >= 0xf000000000000000LL) {
1629  /* DQS is late, needs negative adjustment */
1630  for (i = 0; i < 8; i++)
1631  if (val << (8 * (7 - i) + 4))
1632  return -i;
1633  } else {
1634  /* DQS is early, needs positive adjustment */
1635  for (i = 0; i < 8; i++)
1636  if (val >> (8 * (7 - i) + 4))
1637  return i;
1638  }
1639  return 8;
1640 }
1641 
1642 static void train_write_flyby(ramctr_timing *ctrl)
1643 {
1644  int channel, slotrank, lane, old;
1645 
1646  const union gdcr_training_mod_reg training_mod = {
1647  .dq_dqs_training_res = 1,
1648  };
1649  mchbar_write32(GDCRTRAININGMOD, training_mod.raw);
1650 
1651  FOR_ALL_POPULATED_CHANNELS {
1652  fill_pattern1(ctrl, channel);
1653  }
1654  FOR_ALL_POPULATED_CHANNELS FOR_ALL_POPULATED_RANKS {
1655 
1656  /* Reset read and write WDB pointers */
1657  mchbar_write32(IOSAV_DATA_CTL_ch(channel), 0x10001);
1658 
1659  wait_for_iosav(channel);
1660 
1661  iosav_write_misc_write_sequence(ctrl, channel, slotrank, 3, 1, 3, 3, 31);
1662 
1663  iosav_run_once_and_wait(channel);
1664 
1665  const struct iosav_ssq rd_sequence[] = {
1666  /* DRAM command PREA */
1667  [0] = {
1668  .sp_cmd_ctrl = {
1669  .command = IOSAV_PRE,
1670  .ranksel_ap = 1,
1671  },
1672  .subseq_ctrl = {
1673  .cmd_executions = 1,
1674  .cmd_delay_gap = 3,
1675  .post_ssq_wait = ctrl->tRP,
1676  .data_direction = SSQ_NA,
1677  },
1678  .sp_cmd_addr = {
1679  .address = 1 << 10,
1680  .rowbits = 6,
1681  .bank = 0,
1682  .rank = slotrank,
1683  },
1684  .addr_update = {
1685  .addr_wrap = 18,
1686  },
1687  },
1688  /* DRAM command ACT */
1689  [1] = {
1690  .sp_cmd_ctrl = {
1691  .command = IOSAV_ACT,
1692  .ranksel_ap = 1,
1693  },
1694  .subseq_ctrl = {
1695  .cmd_executions = 1,
1696  .cmd_delay_gap = 3,
1697  .post_ssq_wait = ctrl->tRCD,
1698  .data_direction = SSQ_NA,
1699  },
1700  .sp_cmd_addr = {
1701  .address = 0,
1702  .rowbits = 6,
1703  .bank = 0,
1704  .rank = slotrank,
1705  },
1706  },
1707  /* DRAM command RDA */
1708  [2] = {
1709  .sp_cmd_ctrl = {
1710  .command = IOSAV_RD,
1711  .ranksel_ap = 3,
1712  },
1713  .subseq_ctrl = {
1714  .cmd_executions = 1,
1715  .cmd_delay_gap = 3,
1716  .post_ssq_wait = ctrl->tRP +
1717  ctrl->timings[channel][slotrank].roundtrip_latency +
1718  ctrl->timings[channel][slotrank].io_latency,
1719  .data_direction = SSQ_RD,
1720  },
1721  .sp_cmd_addr = {
1722  .address = 8,
1723  .rowbits = 6,
1724  .bank = 0,
1725  .rank = slotrank,
1726  },
1727  },
1728  };
1729  iosav_write_sequence(channel, rd_sequence, ARRAY_SIZE(rd_sequence));
1730 
1731  iosav_run_once_and_wait(channel);
1732 
1733  FOR_ALL_LANES {
1734  u64 res = mchbar_read32(lane_base[lane] + GDCRTRAININGRESULT1(channel));
1735  res |= ((u64) mchbar_read32(lane_base[lane] +
1736  GDCRTRAININGRESULT2(channel))) << 32;
1737 
1738  old = ctrl->timings[channel][slotrank].lanes[lane].tx_dqs;
1739  ctrl->timings[channel][slotrank].lanes[lane].tx_dqs +=
1740  get_dqs_flyby_adjust(res) * QCLK_PI;
1741 
1742  printram("High adjust %d:%016llx\n", lane, res);
1743  printram("Bval+: %d, %d, %d, % 4d -> % 4d\n", channel, slotrank, lane,
1744  old, ctrl->timings[channel][slotrank].lanes[lane].tx_dqs);
1745  }
1746  }
1747  mchbar_write32(GDCRTRAININGMOD, 0);
1748 }
1749 
1750 static void disable_refresh_machine(ramctr_timing *ctrl)
1751 {
1752  int channel;
1753 
1754  FOR_ALL_POPULATED_CHANNELS {
1755  /* choose an existing rank */
1756  const int slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
1757 
1758  iosav_write_zqcs_sequence(channel, slotrank, 4, 4, 31);
1759 
1760  iosav_run_once_and_wait(channel);
1761 
1762  mchbar_setbits32(SCHED_CBIT_ch(channel), 1 << 21);
1763  }
1764 
1765  /* Refresh disable */
1766  mchbar_clrbits32(MC_INIT_STATE_G, 1 << 3);
1767 
1768  FOR_ALL_POPULATED_CHANNELS {
1769  /* Execute the same command queue */
1770  iosav_run_once_and_wait(channel);
1771  }
1772 }
1773 
1774 /*
1775  * Compensate the skew between CMD/ADDR/CLK and DQ/DQS lanes.
1776  *
1777  * Since DDR3 uses a fly-by topology, the data and strobes signals reach the chips at different
1778  * times with respect to command, address and clock signals. By delaying either all DQ/DQS or
1779  * all CMD/ADDR/CLK signals, a full phase shift can be introduced. It is assumed that the
1780  * CLK/ADDR/CMD signals have the same routing delay.
1781  *
1782  * To find the required phase shift the DRAM is placed in "write leveling" mode. In this mode,
1783  * the DRAM-chip samples the CLK on every DQS edge and feeds back the sampled value on the data
1784  * lanes (DQ).
1785  */
1786 static int jedec_write_leveling(ramctr_timing *ctrl)
1787 {
1788  int channel, slotrank;
1789 
1790  disable_refresh_machine(ctrl);
1791 
1792  /* Enable write leveling on all ranks
1793  Disable all DQ outputs
1794  Only NOP is allowed in this mode */
1795  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
1796  write_mrreg(ctrl, channel, slotrank, 1,
1797  make_mr1(ctrl, slotrank, channel) | 1 << 12 | 1 << 7);
1798 
1799  /* Needs to be programmed before I/O reset below */
1800  const union gdcr_training_mod_reg training_mod = {
1801  .write_leveling_mode = 1,
1802  .enable_dqs_wl = 5,
1803  .odt_always_on = 1,
1804  .force_drive_enable = 1,
1805  };
1806  mchbar_write32(GDCRTRAININGMOD, training_mod.raw);
1807 
1808  toggle_io_reset();
1809 
1810  /* Set any valid value for tx_dqs, it gets corrected later */
1811  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
1812  const int err = write_level_rank(ctrl, channel, slotrank);
1813  if (err)
1814  return err;
1815  }
1816 
1817  /* Disable write leveling on all ranks */
1818  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
1819  write_mrreg(ctrl, channel, slotrank, 1, make_mr1(ctrl, slotrank, channel));
1820 
1821  mchbar_write32(GDCRTRAININGMOD, 0);
1822 
1823  FOR_ALL_POPULATED_CHANNELS
1824  wait_for_iosav(channel);
1825 
1826  /* Refresh enable */
1827  mchbar_setbits32(MC_INIT_STATE_G, 1 << 3);
1828 
1829  FOR_ALL_POPULATED_CHANNELS {
1830  mchbar_clrbits32(SCHED_CBIT_ch(channel), 1 << 21);
1831  mchbar_read32(IOSAV_STATUS_ch(channel));
1832  wait_for_iosav(channel);
1833 
1834  iosav_write_zqcs_sequence(channel, 0, 4, 101, 31);
1835 
1836  iosav_run_once_and_wait(channel);
1837  }
1838 
1839  toggle_io_reset();
1840 
1841  return 0;
1842 }
1843 
1844 int write_training(ramctr_timing *ctrl)
1845 {
1846  int channel, slotrank;
1847  int err;
1848 
1849  /*
1850  * Set the DEC_WRD bit, required for the write flyby algorithm.
1851  * Needs to be done before starting the write training procedure.
1852  */
1853  FOR_ALL_POPULATED_CHANNELS
1854  mchbar_setbits32(TC_RWP_ch(channel), 1 << 27);
1855 
1856  printram("CPE\n");
1857 
1858  err = jedec_write_leveling(ctrl);
1859  if (err)
1860  return err;
1861 
1862  printram("CPF\n");
1863 
1864  FOR_ALL_POPULATED_CHANNELS {
1865  fill_pattern0(ctrl, channel, 0xaaaaaaaa, 0x55555555);
1866  }
1867 
1868  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
1869  err = tx_dq_write_leveling(ctrl, channel, slotrank);
1870  if (err)
1871  return err;
1872  }
1873 
1874  FOR_ALL_POPULATED_CHANNELS
1875  program_timings(ctrl, channel);
1876 
1877  /* measure and adjust tx_dqs timings */
1878  train_write_flyby(ctrl);
1879 
1880  FOR_ALL_POPULATED_CHANNELS
1881  program_timings(ctrl, channel);
1882 
1883  return 0;
1884 }
1885 
1886 static int test_command_training(ramctr_timing *ctrl, int channel, int slotrank)
1887 {
1888  struct ram_rank_timings saved_rt = ctrl->timings[channel][slotrank];
1889  int tx_dq_delta;
1890  int lanes_ok = 0;
1891  int ctr = 0;
1892  int lane;
1893 
1894  for (tx_dq_delta = -5; tx_dq_delta <= 5; tx_dq_delta++) {
1895  FOR_ALL_LANES {
1896  ctrl->timings[channel][slotrank].lanes[lane].tx_dq =
1897  saved_rt.lanes[lane].tx_dq + tx_dq_delta;
1898  }
1899  program_timings(ctrl, channel);
1900  FOR_ALL_LANES {
1901  mchbar_write32(IOSAV_By_ERROR_COUNT(lane), 0);
1902  }
1903 
1904  /* Reset read WDB pointer */
1905  mchbar_write32(IOSAV_DATA_CTL_ch(channel), 0x1f);
1906 
1907  wait_for_iosav(channel);
1908 
1909  iosav_write_command_training_sequence(ctrl, channel, slotrank, ctr);
1910 
1911  /* Program LFSR for the RD/WR subsequences */
1912  mchbar_write32(IOSAV_n_ADDRESS_LFSR_ch(channel, 1), 0x389abcd);
1913  mchbar_write32(IOSAV_n_ADDRESS_LFSR_ch(channel, 2), 0x389abcd);
1914 
1915  iosav_run_once_and_wait(channel);
1916 
1917  FOR_ALL_LANES {
1918  u32 r32 = mchbar_read32(IOSAV_By_ERROR_COUNT_ch(channel, lane));
1919 
1920  if (r32 == 0)
1921  lanes_ok |= 1 << lane;
1922  }
1923  ctr++;
1924  if (lanes_ok == ((1 << ctrl->lanes) - 1))
1925  break;
1926  }
1927 
1928  ctrl->timings[channel][slotrank] = saved_rt;
1929 
1930  return lanes_ok != ((1 << ctrl->lanes) - 1);
1931 }
1932 
1933 static void fill_pattern5(ramctr_timing *ctrl, int channel, int patno)
1934 {
1935  unsigned int i, j;
1936  unsigned int offset = get_precedening_channels(ctrl, channel) * 64;
1937  unsigned int step = 64 * num_of_channels(ctrl);
1938  uintptr_t addr;
1939 
1940  if (patno) {
1941  u8 base8 = 0x80 >> ((patno - 1) % 8);
1942  u32 base = base8 | (base8 << 8) | (base8 << 16) | (base8 << 24);
1943  for (i = 0; i < 32; i++) {
1944  for (j = 0; j < 16; j++) {
1945  u32 val = use_base[patno - 1][i] & (1 << (j / 2)) ? base : 0;
1946 
1947  if (invert[patno - 1][i] & (1 << (j / 2)))
1948  val = ~val;
1949 
1950  addr = (1 << 26) + offset + i * step + j * 4;
1951  write32((void *)addr, val);
1952  }
1953  }
1954  } else {
1955  for (i = 0; i < ARRAY_SIZE(pattern); i++) {
1956  for (j = 0; j < 16; j++) {
1957  const u32 val = pattern[i][j];
1958  addr = (1 << 26) + offset + i * step + j * 4;
1959  write32((void *)addr, val);
1960  }
1961  }
1962  sfence();
1963  }
1964 
1965  program_wdb_pattern_length(channel, 256);
1966 }
1967 
1968 static void reprogram_320c(ramctr_timing *ctrl)
1969 {
1970  disable_refresh_machine(ctrl);
1971 
1972  /* JEDEC reset */
1973  dram_jedecreset(ctrl);
1974 
1975  /* MRS commands */
1976  dram_mrscommands(ctrl);
1977 
1978  toggle_io_reset();
1979 }
1980 
1981 #define CT_MIN_PI (-CCC_MAX_PI)
1982 #define CT_MAX_PI (+CCC_MAX_PI + 1)
1983 #define CT_PI_LENGTH (CT_MAX_PI - CT_MIN_PI + 1)
1984 
1985 #define MIN_C320C_LEN 13
1986 
1987 static int try_cmd_stretch(ramctr_timing *ctrl, int channel, int cmd_stretch)
1988 {
1989  struct ram_rank_timings saved_timings[NUM_CHANNELS][NUM_SLOTRANKS];
1990  int slotrank;
1991  int command_pi;
1992  int stat[NUM_SLOTRANKS][CT_PI_LENGTH];
1993  int delta = 0;
1994 
1995  printram("Trying cmd_stretch %d on channel %d\n", cmd_stretch, channel);
1996 
1997  FOR_ALL_POPULATED_RANKS {
1998  saved_timings[channel][slotrank] = ctrl->timings[channel][slotrank];
1999  }
2000 
2001  ctrl->cmd_stretch[channel] = cmd_stretch;
2002 
2003  const union tc_rap_reg tc_rap = {
2004  .tRRD = ctrl->tRRD,
2005  .tRTP = ctrl->tRTP,
2006  .tCKE = ctrl->tCKE,
2007  .tWTR = ctrl->tWTR,
2008  .tFAW = ctrl->tFAW,
2009  .tWR = ctrl->tWR,
2010  .tCMD = ctrl->cmd_stretch[channel],
2011  };
2012  mchbar_write32(TC_RAP_ch(channel), tc_rap.raw);
2013 
2014  if (ctrl->cmd_stretch[channel] == 2)
2015  delta = 2;
2016  else if (ctrl->cmd_stretch[channel] == 0)
2017  delta = 4;
2018 
2019  FOR_ALL_POPULATED_RANKS {
2020  ctrl->timings[channel][slotrank].roundtrip_latency -= delta;
2021  }
2022 
2023  for (command_pi = CT_MIN_PI; command_pi < CT_MAX_PI; command_pi++) {
2024  FOR_ALL_POPULATED_RANKS {
2025  ctrl->timings[channel][slotrank].pi_coding = command_pi;
2026  }
2027  program_timings(ctrl, channel);
2028  reprogram_320c(ctrl);
2029  FOR_ALL_POPULATED_RANKS {
2030  stat[slotrank][command_pi - CT_MIN_PI] =
2031  test_command_training(ctrl, channel, slotrank);
2032  }
2033  }
2034  FOR_ALL_POPULATED_RANKS {
2035  struct run rn = get_longest_zero_run(stat[slotrank], CT_PI_LENGTH - 1);
2036 
2037  ctrl->timings[channel][slotrank].pi_coding = rn.middle + CT_MIN_PI;
2038  printram("cmd_stretch: %d, %d: % 4d-% 4d-% 4d\n",
2039  channel, slotrank, rn.start, rn.middle, rn.end);
2040 
2041  if (rn.all || rn.length < MIN_C320C_LEN) {
2042  FOR_ALL_POPULATED_RANKS {
2043  ctrl->timings[channel][slotrank] =
2044  saved_timings[channel][slotrank];
2045  }
2046  return MAKE_ERR;
2047  }
2048  }
2049 
2050  return 0;
2051 }
2052 
2053 /*
2054  * Adjust CMD phase shift and try multiple command rates.
2055  * A command rate of 2T doubles the time needed for address and command decode.
2056  */
2057 int command_training(ramctr_timing *ctrl)
2058 {
2059  int channel;
2060 
2061  FOR_ALL_POPULATED_CHANNELS {
2062  fill_pattern5(ctrl, channel, 0);
2063  }
2064 
2065  FOR_ALL_POPULATED_CHANNELS {
2066  int cmdrate, err;
2067 
2068  /*
2069  * Dual DIMM per channel:
2070  * Issue:
2071  * While command training seems to succeed, raminit will fail in write training.
2072  *
2073  * Workaround:
2074  * Skip 1T in dual DIMM mode, that's only supported by a few DIMMs.
2075  * Only try 1T mode for XMP DIMMs that request it in dual DIMM mode.
2076  *
2077  * Single DIMM per channel:
2078  * Try command rate 1T and 2T
2079  */
2080  cmdrate = ((ctrl->rankmap[channel] & 0x5) == 0x5);
2081  if (ctrl->tCMD)
2082  /* XMP gives the CMD rate in clock ticks, not ns */
2083  cmdrate = MIN(DIV_ROUND_UP(ctrl->tCMD, 256) - 1, 1);
2084 
2085  for (; cmdrate < 2; cmdrate++) {
2086  err = try_cmd_stretch(ctrl, channel, cmdrate << 1);
2087 
2088  if (!err)
2089  break;
2090  }
2091 
2092  if (err) {
2093  printk(BIOS_EMERG, "Command training failed: %d\n", channel);
2094  return err;
2095  }
2096 
2097  printram("Using CMD rate %uT on channel %u\n", cmdrate + 1, channel);
2098  }
2099 
2100  FOR_ALL_POPULATED_CHANNELS
2101  program_timings(ctrl, channel);
2102 
2103  reprogram_320c(ctrl);
2104  return 0;
2105 }
2106 
2107 static int find_read_mpr_margin(ramctr_timing *ctrl, int channel, int slotrank, int *edges)
2108 {
2109  int dqs_pi;
2110  int stats[NUM_LANES][MAX_EDGE_TIMING + 1];
2111  int lane;
2112 
2113  for (dqs_pi = 0; dqs_pi <= MAX_EDGE_TIMING; dqs_pi++) {
2114  FOR_ALL_LANES {
2115  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_p = dqs_pi;
2116  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_n = dqs_pi;
2117  }
2118  program_timings(ctrl, channel);
2119 
2120  FOR_ALL_LANES {
2121  mchbar_write32(IOSAV_By_ERROR_COUNT_ch(channel, lane), 0);
2122  mchbar_read32(IOSAV_By_BW_SERROR_C_ch(channel, lane));
2123  }
2124 
2125  wait_for_iosav(channel);
2126 
2127  iosav_write_read_mpr_sequence(
2128  channel, slotrank, ctrl->tMOD, 500, 4, 1, ctrl->CAS + 8);
2129 
2130  iosav_run_once_and_wait(channel);
2131 
2132  FOR_ALL_LANES {
2133  stats[lane][dqs_pi] = mchbar_read32(
2134  IOSAV_By_ERROR_COUNT_ch(channel, lane));
2135  }
2136  }
2137 
2138  FOR_ALL_LANES {
2139  struct run rn = get_longest_zero_run(stats[lane], MAX_EDGE_TIMING + 1);
2140  edges[lane] = rn.middle;
2141 
2142  if (rn.all) {
2143  printk(BIOS_EMERG, "Read MPR training failed: %d, %d, %d\n", channel,
2144  slotrank, lane);
2145  return MAKE_ERR;
2146  }
2147  printram("eval %d, %d, %d: % 4d\n", channel, slotrank, lane, edges[lane]);
2148  }
2149  return 0;
2150 }
2151 
2152 static void find_predefined_pattern(ramctr_timing *ctrl, const int channel)
2153 {
2154  int slotrank, lane;
2155 
2156  fill_pattern0(ctrl, channel, 0, 0);
2157  FOR_ALL_LANES {
2158  mchbar_write32(IOSAV_By_BW_MASK_ch(channel, lane), 0);
2159  mchbar_read32(IOSAV_By_BW_SERROR_C_ch(channel, lane));
2160  }
2161 
2162  FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2163  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_n = 16;
2164  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_p = 16;
2165  }
2166 
2167  program_timings(ctrl, channel);
2168 
2169  FOR_ALL_POPULATED_RANKS {
2170  wait_for_iosav(channel);
2171 
2172  iosav_write_read_mpr_sequence(
2173  channel, slotrank, ctrl->tMOD, 3, 4, 1, ctrl->CAS + 8);
2174 
2175  iosav_run_once_and_wait(channel);
2176  }
2177 
2178  /* XXX: check any measured value ? */
2179 
2180  FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2181  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_n = 48;
2182  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_p = 48;
2183  }
2184 
2185  program_timings(ctrl, channel);
2186 
2187  FOR_ALL_POPULATED_RANKS {
2188  wait_for_iosav(channel);
2189 
2190  iosav_write_read_mpr_sequence(
2191  channel, slotrank, ctrl->tMOD, 3, 4, 1, ctrl->CAS + 8);
2192 
2193  iosav_run_once_and_wait(channel);
2194  }
2195 
2196  /* XXX: check any measured value ? */
2197 
2198  FOR_ALL_LANES {
2199  mchbar_write32(IOSAV_By_BW_MASK_ch(channel, lane),
2200  ~mchbar_read32(IOSAV_By_BW_SERROR_ch(channel, lane)) & 0xff);
2201  }
2202 }
2203 
2204 int read_mpr_training(ramctr_timing *ctrl)
2205 {
2206  int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2207  int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2208  int channel, slotrank, lane;
2209  int err;
2210 
2211  mchbar_write32(GDCRTRAININGMOD, 0);
2212 
2213  toggle_io_reset();
2214 
2215  FOR_ALL_POPULATED_CHANNELS {
2216  find_predefined_pattern(ctrl, channel);
2217 
2218  fill_pattern0(ctrl, channel, 0, 0xffffffff);
2219  }
2220 
2221  /*
2222  * FIXME: Under some conditions, vendor BIOS sets both edges to the same value. It will
2223  * also use a single loop. It would seem that it is a debugging configuration.
2224  */
2225  mchbar_write32(IOSAV_DC_MASK, 3 << 8);
2226  printram("discover falling edges:\n[%x] = %x\n", IOSAV_DC_MASK, 3 << 8);
2227 
2228  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2229  err = find_read_mpr_margin(ctrl, channel, slotrank,
2230  falling_edges[channel][slotrank]);
2231  if (err)
2232  return err;
2233  }
2234 
2235  mchbar_write32(IOSAV_DC_MASK, 2 << 8);
2236  printram("discover rising edges:\n[%x] = %x\n", IOSAV_DC_MASK, 2 << 8);
2237 
2238  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2239  err = find_read_mpr_margin(ctrl, channel, slotrank,
2240  rising_edges[channel][slotrank]);
2241  if (err)
2242  return err;
2243  }
2244 
2245  mchbar_write32(IOSAV_DC_MASK, 0);
2246 
2247  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2248  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_n =
2249  falling_edges[channel][slotrank][lane];
2250  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_p =
2251  rising_edges[channel][slotrank][lane];
2252  }
2253 
2254  FOR_ALL_POPULATED_CHANNELS {
2255  program_timings(ctrl, channel);
2256  }
2257 
2258  FOR_ALL_POPULATED_CHANNELS FOR_ALL_LANES {
2259  mchbar_write32(IOSAV_By_BW_MASK_ch(channel, lane), 0);
2260  }
2261  return 0;
2262 }
2263 
2264 static int find_agrsv_read_margin(ramctr_timing *ctrl, int channel, int slotrank, int *edges)
2265 {
2266  const int rd_vref_offsets[] = { 0, 0xc, 0x2c };
2267 
2268  u32 raw_stats[MAX_EDGE_TIMING + 1];
2269  int lower[NUM_LANES];
2270  int upper[NUM_LANES];
2271  int lane, i, read_pi, pat;
2272 
2273  FOR_ALL_LANES {
2274  lower[lane] = 0;
2275  upper[lane] = MAX_EDGE_TIMING;
2276  }
2277 
2278  for (i = 0; i < ARRAY_SIZE(rd_vref_offsets); i++) {
2279  const union gdcr_training_mod_reg training_mod = {
2280  .vref_gen_ctl = rd_vref_offsets[i],
2281  };
2282  mchbar_write32(GDCRTRAININGMOD_ch(channel), training_mod.raw);
2283  printram("[%x] = 0x%08x\n", GDCRTRAININGMOD_ch(channel), training_mod.raw);
2284 
2285  for (pat = 0; pat < NUM_PATTERNS; pat++) {
2286  fill_pattern5(ctrl, channel, pat);
2287  printram("using pattern %d\n", pat);
2288 
2289  for (read_pi = 0; read_pi <= MAX_EDGE_TIMING; read_pi++) {
2290  FOR_ALL_LANES {
2291  ctrl->timings[channel][slotrank].lanes[lane]
2292  .rx_dqs_p = read_pi;
2293  ctrl->timings[channel][slotrank].lanes[lane]
2294  .rx_dqs_n = read_pi;
2295  }
2296  program_timings(ctrl, channel);
2297 
2298  FOR_ALL_LANES {
2299  mchbar_write32(IOSAV_By_ERROR_COUNT_ch(channel, lane),
2300  0);
2301  mchbar_read32(IOSAV_By_BW_SERROR_C_ch(channel, lane));
2302  }
2303  wait_for_iosav(channel);
2304 
2305  iosav_write_data_write_sequence(ctrl, channel, slotrank);
2306 
2307  iosav_run_once_and_wait(channel);
2308 
2309  FOR_ALL_LANES {
2310  mchbar_read32(IOSAV_By_ERROR_COUNT_ch(channel, lane));
2311  }
2312 
2313  /* FIXME: This register only exists on Ivy Bridge */
2314  raw_stats[read_pi] = mchbar_read32(
2315  IOSAV_BYTE_SERROR_C_ch(channel));
2316  }
2317 
2318  FOR_ALL_LANES {
2319  int stats[MAX_EDGE_TIMING + 1];
2320  struct run rn;
2321 
2322  for (read_pi = 0; read_pi <= MAX_EDGE_TIMING; read_pi++)
2323  stats[read_pi] = !!(raw_stats[read_pi] & (1 << lane));
2324 
2325  rn = get_longest_zero_run(stats, MAX_EDGE_TIMING + 1);
2326 
2327  printram("edges: %d, %d, %d: % 4d-% 4d-% 4d, "
2328  "% 4d-% 4d\n", channel, slotrank, i, rn.start,
2329  rn.middle, rn.end, rn.start + ctrl->edge_offset[i],
2330  rn.end - ctrl->edge_offset[i]);
2331 
2332  lower[lane] = MAX(rn.start + ctrl->edge_offset[i], lower[lane]);
2333  upper[lane] = MIN(rn.end - ctrl->edge_offset[i], upper[lane]);
2334 
2335  edges[lane] = (lower[lane] + upper[lane]) / 2;
2336  if (rn.all || (lower[lane] > upper[lane])) {
2337  printk(BIOS_EMERG, "Aggressive read training failed: "
2338  "%d, %d, %d\n", channel, slotrank, lane);
2339 
2340  return MAKE_ERR;
2341  }
2342  }
2343  }
2344  }
2345 
2346  /* Restore nominal Vref after training */
2347  mchbar_write32(GDCRTRAININGMOD_ch(channel), 0);
2348  printram("CPA\n");
2349  return 0;
2350 }
2351 
2352 int aggressive_read_training(ramctr_timing *ctrl)
2353 {
2354  int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2355  int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2356  int channel, slotrank, lane, err;
2357 
2358  /*
2359  * FIXME: Under some conditions, vendor BIOS sets both edges to the same value. It will
2360  * also use a single loop. It would seem that it is a debugging configuration.
2361  */
2362  mchbar_write32(IOSAV_DC_MASK, 3 << 8);
2363  printram("discover falling edges aggressive:\n[%x] = %x\n", IOSAV_DC_MASK, 3 << 8);
2364 
2365  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2366  err = find_agrsv_read_margin(ctrl, channel, slotrank,
2367  falling_edges[channel][slotrank]);
2368  if (err)
2369  return err;
2370  }
2371 
2372  mchbar_write32(IOSAV_DC_MASK, 2 << 8);
2373  printram("discover rising edges aggressive:\n[%x] = %x\n", IOSAV_DC_MASK, 2 << 8);
2374 
2375  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2376  err = find_agrsv_read_margin(ctrl, channel, slotrank,
2377  rising_edges[channel][slotrank]);
2378  if (err)
2379  return err;
2380  }
2381 
2382  mchbar_write32(IOSAV_DC_MASK, 0);
2383 
2384  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2385  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_n =
2386  falling_edges[channel][slotrank][lane];
2387 
2388  ctrl->timings[channel][slotrank].lanes[lane].rx_dqs_p =
2389  rising_edges[channel][slotrank][lane];
2390  }
2391 
2392  FOR_ALL_POPULATED_CHANNELS
2393  program_timings(ctrl, channel);
2394 
2395  return 0;
2396 }
2397 
2398 static void test_aggressive_write(ramctr_timing *ctrl, int channel, int slotrank)
2399 {
2400  wait_for_iosav(channel);
2401 
2402  iosav_write_aggressive_write_read_sequence(ctrl, channel, slotrank);
2403 
2404  iosav_run_once_and_wait(channel);
2405 }
2406 
2407 static void set_write_vref(const int channel, const u8 wr_vref)
2408 {
2409  mchbar_clrsetbits32(GDCRCMDDEBUGMUXCFG_Cz_S(channel), 0x3f << 24, wr_vref << 24);
2410  udelay(2);
2411 }
2412 
2413 int aggressive_write_training(ramctr_timing *ctrl)
2414 {
2415  const u8 wr_vref_offsets[3] = { 0, 0x0f, 0x2f };
2416  int i, pat;
2417 
2418  int lower[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2419  int upper[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2420  int channel, slotrank, lane;
2421 
2422  /* Changing the write Vref is only supported on some Ivy Bridge SKUs */
2423  if (!IS_IVY_CPU(ctrl->cpu))
2424  return 0;
2425 
2426  if (!(pci_read_config32(HOST_BRIDGE, CAPID0_A) & CAPID_WRTVREF))
2427  return 0;
2428 
2429  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2430  lower[channel][slotrank][lane] = 0;
2431  upper[channel][slotrank][lane] = MAX_TX_DQ;
2432  }
2433 
2434  /* Only enable IOSAV_n_SPECIAL_COMMAND_ADDR optimization on later steppings */
2435  const bool enable_iosav_opt = IS_IVY_CPU_D(ctrl->cpu) || IS_IVY_CPU_E(ctrl->cpu);
2436 
2437  if (enable_iosav_opt)
2438  mchbar_write32(MCMNTS_SPARE, 1);
2439 
2440  printram("Aggressive write training:\n");
2441 
2442  for (i = 0; i < ARRAY_SIZE(wr_vref_offsets); i++) {
2443  FOR_ALL_POPULATED_CHANNELS {
2444  set_write_vref(channel, wr_vref_offsets[i]);
2445 
2446  for (pat = 0; pat < NUM_PATTERNS; pat++) {
2447  FOR_ALL_POPULATED_RANKS {
2448  int tx_dq;
2449  u32 raw_stats[MAX_TX_DQ + 1];
2450  int stats[MAX_TX_DQ + 1];
2451 
2452  /* Make sure rn.start < rn.end */
2453  stats[MAX_TX_DQ] = 1;
2454 
2455  fill_pattern5(ctrl, channel, pat);
2456 
2457  for (tx_dq = 0; tx_dq < MAX_TX_DQ; tx_dq++) {
2458  FOR_ALL_LANES {
2459  ctrl->timings[channel][slotrank]
2460  .lanes[lane].tx_dq = tx_dq;
2461  }
2462  program_timings(ctrl, channel);
2463 
2464  test_aggressive_write(ctrl, channel, slotrank);
2465 
2466  raw_stats[tx_dq] = mchbar_read32(
2467  IOSAV_BYTE_SERROR_C_ch(channel));
2468  }
2469  FOR_ALL_LANES {
2470  struct run rn;
2471  for (tx_dq = 0; tx_dq < MAX_TX_DQ; tx_dq++) {
2472  stats[tx_dq] = !!(raw_stats[tx_dq]
2473  & (1 << lane));
2474  }
2475 
2476  rn = get_longest_zero_run(stats, MAX_TX_DQ + 1);
2477  if (rn.all) {
2478  printk(BIOS_EMERG, "Aggressive "
2479  "write training failed: "
2480  "%d, %d, %d\n", channel,
2481  slotrank, lane);
2482 
2483  return MAKE_ERR;
2484  }
2485  printram("tx_dq: %d, %d, %d: "
2486  "% 4d-% 4d-% 4d, "
2487  "% 4d-% 4d\n", channel, slotrank,
2488  i, rn.start, rn.middle, rn.end,
2489  rn.start + ctrl->tx_dq_offset[i],
2490  rn.end - ctrl->tx_dq_offset[i]);
2491 
2492  lower[channel][slotrank][lane] =
2493  MAX(rn.start + ctrl->tx_dq_offset[i],
2494  lower[channel][slotrank][lane]);
2495 
2496  upper[channel][slotrank][lane] =
2497  MIN(rn.end - ctrl->tx_dq_offset[i],
2498  upper[channel][slotrank][lane]);
2499 
2500  }
2501  }
2502  }
2503  }
2504  }
2505 
2506  FOR_ALL_CHANNELS {
2507  /* Restore nominal write Vref after training */
2508  set_write_vref(channel, 0);
2509  }
2510 
2511  /* Disable IOSAV_n_SPECIAL_COMMAND_ADDR optimization */
2512  if (enable_iosav_opt)
2513  mchbar_write32(MCMNTS_SPARE, 0);
2514 
2515  printram("CPB\n");
2516 
2517  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2518  printram("tx_dq %d, %d, %d: % 4d\n", channel, slotrank, lane,
2519  (lower[channel][slotrank][lane] +
2520  upper[channel][slotrank][lane]) / 2);
2521 
2522  ctrl->timings[channel][slotrank].lanes[lane].tx_dq =
2523  (lower[channel][slotrank][lane] +
2524  upper[channel][slotrank][lane]) / 2;
2525  }
2526  FOR_ALL_POPULATED_CHANNELS {
2527  program_timings(ctrl, channel);
2528  }
2529  return 0;
2530 }
2531 
2532 void normalize_training(ramctr_timing *ctrl)
2533 {
2534  int channel, slotrank, lane;
2535  int mat;
2536 
2537  FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2538  int delta;
2539  mat = 0;
2540  FOR_ALL_LANES mat =
2541  MAX(ctrl->timings[channel][slotrank].lanes[lane].rcven, mat);
2542  printram("normalize %d, %d, %d: mat %d\n",
2543  channel, slotrank, lane, mat);
2544 
2545  delta = (mat >> 6) - ctrl->timings[channel][slotrank].io_latency;
2546  printram("normalize %d, %d, %d: delta %d\n",
2547  channel, slotrank, lane, delta);
2548 
2549  ctrl->timings[channel][slotrank].roundtrip_latency += delta;
2550  ctrl->timings[channel][slotrank].io_latency += delta;
2551  }
2552 
2553  FOR_ALL_POPULATED_CHANNELS {
2554  program_timings(ctrl, channel);
2555  }
2556 }
2557 
2558 int channel_test(ramctr_timing *ctrl)
2559 {
2560  int channel, slotrank, lane;
2561 
2562  slotrank = 0;
2563  FOR_ALL_POPULATED_CHANNELS
2564  if (mchbar_read32(MC_INIT_STATE_ch(channel)) & 0xa000) {
2565  printk(BIOS_EMERG, "Mini channel test failed (1): %d\n", channel);
2566  return MAKE_ERR;
2567  }
2568  FOR_ALL_POPULATED_CHANNELS {
2569  fill_pattern0(ctrl, channel, 0x12345678, 0x98765432);
2570  }
2571 
2572  for (slotrank = 0; slotrank < 4; slotrank++)
2573  FOR_ALL_CHANNELS
2574  if (ctrl->rankmap[channel] & (1 << slotrank)) {
2575  FOR_ALL_LANES {
2576  mchbar_write32(IOSAV_By_ERROR_COUNT(lane), 0);
2577  mchbar_write32(IOSAV_By_BW_SERROR_C(lane), 0);
2578  }
2579  wait_for_iosav(channel);
2580 
2581  iosav_write_memory_test_sequence(ctrl, channel, slotrank);
2582 
2583  iosav_run_once_and_wait(channel);
2584 
2585  FOR_ALL_LANES
2586  if (mchbar_read32(IOSAV_By_ERROR_COUNT_ch(channel, lane))) {
2587  printk(BIOS_EMERG, "Mini channel test failed (2): %d, %d, %d\n",
2588  channel, slotrank, lane);
2589  return MAKE_ERR;
2590  }
2591  }
2592  return 0;
2593 }
2594 
2595 void channel_scrub(ramctr_timing *ctrl)
2596 {
2597  int channel, slotrank, row, rowsize;
2598  u8 bank;
2599 
2600  FOR_ALL_POPULATED_CHANNELS {
2601  wait_for_iosav(channel);
2602  fill_pattern0(ctrl, channel, 0, 0);
2603  }
2604 
2605  /*
2606  * During runtime the "scrubber" will periodically scan through the memory in the
2607  * physical address space, to identify and fix CRC errors.
2608  * The following loops writes to every DRAM address, setting the ECC bits to the
2609  * correct value. A read from this location will no longer return a CRC error,
2610  * except when a bit has toggled due to external events.
2611  * The same could be achieved by writing to the physical memory map, but it's
2612  * much more difficult due to SMM remapping, ME stolen memory, GFX stolen memory,
2613  * and firmware running in x86_32.
2614  */
2615  FOR_ALL_POPULATED_CHANNELS FOR_ALL_POPULATED_RANKS {
2616  rowsize = 1 << ctrl->info.dimm[channel][slotrank >> 1].row_bits;
2617  for (bank = 0; bank < 8; bank++) {
2618  for (row = 0; row < rowsize; row += 16) {
2619 
2620  u8 gap = MAX((ctrl->tFAW >> 2) + 1, ctrl->tRRD);
2621  const struct iosav_ssq sequence[] = {
2622  /*
2623  * DRAM command ACT
2624  * Opens the row for writing.
2625  */
2626  [0] = {
2627  .sp_cmd_ctrl = {
2628  .command = IOSAV_ACT,
2629  .ranksel_ap = 1,
2630  },
2631  .subseq_ctrl = {
2632  .cmd_executions = 1,
2633  .cmd_delay_gap = gap,
2634  .post_ssq_wait = ctrl->tRCD,
2635  .data_direction = SSQ_NA,
2636  },
2637  .sp_cmd_addr = {
2638  .address = row,
2639  .rowbits = 6,
2640  .bank = bank,
2641  .rank = slotrank,
2642  },
2643  .addr_update = {
2644  .inc_addr_1 = 1,
2645  .addr_wrap = 18,
2646  },
2647  },
2648  /*
2649  * DRAM command WR
2650  * Writes (128 + 1) * 8 (burst length) * 8 (bus width)
2651  * bytes.
2652  */
2653  [1] = {
2654  .sp_cmd_ctrl = {
2655  .command = IOSAV_WR,
2656  .ranksel_ap = 1,
2657  },
2658  .subseq_ctrl = {
2659  .cmd_executions = 129,
2660  .cmd_delay_gap = 4,
2661  .post_ssq_wait = ctrl->tWTR +
2662  ctrl->CWL + 8,
2663  .data_direction = SSQ_WR,
2664  },
2665  .sp_cmd_addr = {
2666  .address = row,
2667  .rowbits = 0,
2668  .bank = bank,
2669  .rank = slotrank,
2670  },
2671  .addr_update = {
2672  .inc_addr_8 = 1,
2673  .addr_wrap = 9,
2674  },
2675  },
2676  /*
2677  * DRAM command PRE
2678  * Closes the row.
2679  */
2680  [2] = {
2681  .sp_cmd_ctrl = {
2682  .command = IOSAV_PRE,
2683  .ranksel_ap = 1,
2684  },
2685  .subseq_ctrl = {
2686  .cmd_executions = 1,
2687  .cmd_delay_gap = 4,
2688  .post_ssq_wait = ctrl->tRP,
2689  .data_direction = SSQ_NA,
2690  },
2691  .sp_cmd_addr = {
2692  .address = 0,
2693  .rowbits = 6,
2694  .bank = bank,
2695  .rank = slotrank,
2696  },
2697  .addr_update = {
2698  .addr_wrap = 18,
2699  },
2700  },
2701  };
2702  iosav_write_sequence(channel, sequence, ARRAY_SIZE(sequence));
2703 
2704  iosav_run_queue(channel, 16, 0);
2705 
2706  wait_for_iosav(channel);
2707  }
2708  }
2709  }
2710 }
2711 
2712 void set_scrambling_seed(ramctr_timing *ctrl)
2713 {
2714  int channel;
2715 
2716  /* FIXME: we hardcode seeds. Do we need to use some PRNG for them? I don't think so. */
2717  static u32 seeds[NUM_CHANNELS][3] = {
2718  {0x00009a36, 0xbafcfdcf, 0x46d1ab68},
2719  {0x00028bfa, 0x53fe4b49, 0x19ed5483}
2720  };
2721  FOR_ALL_POPULATED_CHANNELS {
2722  mchbar_clrbits32(SCHED_CBIT_ch(channel), 1 << 28);
2723  mchbar_write32(SCRAMBLING_SEED_1_ch(channel), seeds[channel][0]);
2724  mchbar_write32(SCRAMBLING_SEED_2_HI_ch(channel), seeds[channel][1]);
2725  mchbar_write32(SCRAMBLING_SEED_2_LO_ch(channel), seeds[channel][2]);
2726  }
2727 }
2728 
2729 void set_wmm_behavior(const u32 cpu)
2730 {
2731  if (IS_SANDY_CPU(cpu) && (IS_SANDY_CPU_D0(cpu) || IS_SANDY_CPU_D1(cpu))) {
2732  mchbar_write32(SC_WDBWM, 0x141d1519);
2733  } else {
2734  mchbar_write32(SC_WDBWM, 0x551d1519);
2735  }
2736 }
2737 
2738 void prepare_training(ramctr_timing *ctrl)
2739 {
2740  int channel;
2741 
2742  FOR_ALL_POPULATED_CHANNELS {
2743  /* Always drive command bus */
2744  mchbar_setbits32(TC_RAP_ch(channel), 1 << 29);
2745  }
2746 
2747  udelay(1);
2748 
2749  FOR_ALL_POPULATED_CHANNELS {
2750  wait_for_iosav(channel);
2751  }
2752 }
2753 
2754 void set_read_write_timings(ramctr_timing *ctrl)
2755 {
2756  /* Use a larger delay when running fast to improve stability */
2757  const u32 tRWDRDD_inc = ctrl->tCK <= TCK_1066MHZ ? 4 : 2;
2758 
2759  int channel, slotrank;
2760 
2761  FOR_ALL_POPULATED_CHANNELS {
2762  int min_pi = 10000;
2763  int max_pi = -10000;
2764 
2765  FOR_ALL_POPULATED_RANKS {
2766  max_pi = MAX(ctrl->timings[channel][slotrank].pi_coding, max_pi);
2767  min_pi = MIN(ctrl->timings[channel][slotrank].pi_coding, min_pi);
2768  }
2769 
2770  const u32 tWRDRDD = (max_pi - min_pi > 51) ? 0 : ctrl->ref_card_offset[channel];
2771 
2772  const u32 val = (ctrl->pi_coding_threshold < max_pi - min_pi) ? 3 : 2;
2773 
2774  dram_odt_stretch(ctrl, channel);
2775 
2776  const union tc_rwp_reg tc_rwp = {
2777  .tRRDR = 0,
2778  .tRRDD = val,
2779  .tWWDR = val,
2780  .tWWDD = val,
2781  .tRWDRDD = ctrl->ref_card_offset[channel] + tRWDRDD_inc,
2782  .tWRDRDD = tWRDRDD,
2783  .tRWSR = 2,
2784  .dec_wrd = 1,
2785  };
2786  mchbar_write32(TC_RWP_ch(channel), tc_rwp.raw);
2787  }
2788 }
2789 
2790 void set_normal_operation(ramctr_timing *ctrl)
2791 {
2792  int channel;
2793  FOR_ALL_POPULATED_CHANNELS {
2794  mchbar_write32(MC_INIT_STATE_ch(channel), 1 << 12 | ctrl->rankmap[channel]);
2795  mchbar_clrbits32(TC_RAP_ch(channel), 1 << 29);
2796  }
2797 }
2798 
2799 /* Encode the watermark latencies in a suitable format for graphics drivers consumption */
2800 static int encode_wm(int ns)
2801 {
2802  return (ns + 499) / 500;
2803 }
2804 
2805 /* FIXME: values in this function should be hardware revision-dependent */
2806 void final_registers(ramctr_timing *ctrl)
2807 {
2808  int channel;
2809  int t1_cycles = 0, t1_ns = 0, t2_ns;
2810  int t3_ns;
2811  u32 r32;
2812 
2813  /* FIXME: This register only exists on Ivy Bridge */
2814  mchbar_write32(WMM_READ_CONFIG, 0x46);
2815 
2816  FOR_ALL_CHANNELS {
2817  union tc_othp_reg tc_othp = {
2818  .raw = mchbar_read32(TC_OTHP_ch(channel)),
2819  };
2820  tc_othp.tCPDED = 1;
2821  mchbar_write32(TC_OTHP_ch(channel), tc_othp.raw);
2822  }
2823 
2824  /* 64 DCLKs until idle, decision per rank */
2825  mchbar_write32(PM_PDWN_CONFIG, get_power_down_mode(ctrl) << 8 | 64);
2826 
2827  FOR_ALL_CHANNELS
2828  mchbar_write32(PM_TRML_M_CONFIG_ch(channel), 0x00000aaa);
2829 
2830  mchbar_write32(PM_BW_LIMIT_CONFIG, 0x5f7003ff);
2831  mchbar_write32(PM_DLL_CONFIG, 0x00073000 | ctrl->mdll_wake_delay);
2832 
2833  FOR_ALL_CHANNELS {
2834  switch (ctrl->rankmap[channel]) {
2835  /* Unpopulated channel */
2836  case 0:
2837  mchbar_write32(PM_CMD_PWR_ch(channel), 0);
2838  break;
2839  /* Only single-ranked dimms */
2840  case 1:
2841  case 4:
2842  case 5:
2843  mchbar_write32(PM_CMD_PWR_ch(channel), 0x00373131);
2844  break;
2845  /* Dual-ranked dimms present */
2846  default:
2847  mchbar_write32(PM_CMD_PWR_ch(channel), 0x009b6ea1);
2848  break;
2849  }
2850  }
2851 
2852  mchbar_write32(MEM_TRML_ESTIMATION_CONFIG, 0xca9171e5);
2853  mchbar_clrsetbits32(MEM_TRML_THRESHOLDS_CONFIG, 0x00ffffff, 0x00e4d5d0);
2854  mchbar_clrbits32(MEM_TRML_INTERRUPT, 0x1f);
2855 
2856  FOR_ALL_CHANNELS {
2857  union tc_rfp_reg tc_rfp = {
2858  .raw = mchbar_read32(TC_RFP_ch(channel)),
2859  };
2860  tc_rfp.refresh_2x_control = 1;
2861  mchbar_write32(TC_RFP_ch(channel), tc_rfp.raw);
2862  }
2863 
2864  mchbar_setbits32(MC_INIT_STATE_G, 1 << 0);
2865  mchbar_setbits32(MC_INIT_STATE_G, 1 << 7);
2866  mchbar_write32(BANDTIMERS_SNB, 0xfa);
2867 
2868  /* Find a populated channel */
2869  FOR_ALL_POPULATED_CHANNELS
2870  break;
2871 
2872  t1_cycles = (mchbar_read32(TC_ZQCAL_ch(channel)) >> 8) & 0xff;
2873  r32 = mchbar_read32(PM_DLL_CONFIG);
2874  if (r32 & (1 << 17))
2875  t1_cycles += (r32 & 0xfff);
2876  t1_cycles += mchbar_read32(TC_SRFTP_ch(channel)) & 0xfff;
2877  t1_ns = t1_cycles * ctrl->tCK / 256 + 544;
2878  if (!(r32 & (1 << 17)))
2879  t1_ns += 500;
2880 
2881  t2_ns = 10 * ((mchbar_read32(SAPMTIMERS) >> 8) & 0xfff);
2882  if (mchbar_read32(SAPMCTL) & 8) {
2883  t3_ns = 10 * ((mchbar_read32(BANDTIMERS_IVB) >> 8) & 0xfff);
2884  t3_ns += 10 * (mchbar_read32(SAPMTIMERS2_IVB) & 0xff);
2885  } else {
2886  t3_ns = 500;
2887  }
2888 
2889  /* The graphics driver will use these watermark values */
2890  printk(BIOS_DEBUG, "t123: %d, %d, %d\n", t1_ns, t2_ns, t3_ns);
2891  mchbar_clrsetbits32(SSKPD, 0x3f3f3f3f,
2892  ((encode_wm(t1_ns) + encode_wm(t2_ns)) << 16) | (encode_wm(t1_ns) << 8) |
2893  ((encode_wm(t3_ns) + encode_wm(t2_ns) + encode_wm(t1_ns)) << 24) | 0x0c);
2894 }
2895 
2896 void restore_timings(ramctr_timing *ctrl)
2897 {
2898  int channel;
2899 
2900  FOR_ALL_POPULATED_CHANNELS {
2901  const union tc_rap_reg tc_rap = {
2902  .tRRD = ctrl->tRRD,
2903  .tRTP = ctrl->tRTP,
2904  .tCKE = ctrl->tCKE,
2905  .tWTR = ctrl->tWTR,
2906  .tFAW = ctrl->tFAW,
2907  .tWR = ctrl->tWR,
2908  .tCMD = ctrl->cmd_stretch[channel],
2909  };
2910  mchbar_write32(TC_RAP_ch(channel), tc_rap.raw);
2911  }
2912 
2913  udelay(1);
2914 
2915  FOR_ALL_POPULATED_CHANNELS {
2916  wait_for_iosav(channel);
2917  }
2918 
2919  FOR_ALL_POPULATED_CHANNELS
2920  mchbar_setbits32(TC_RWP_ch(channel), 1 << 27);
2921 
2922  FOR_ALL_POPULATED_CHANNELS {
2923  udelay(1);
2924  mchbar_setbits32(SCHED_CBIT_ch(channel), 1 << 21);
2925  }
2926 
2927  printram("CPE\n");
2928 
2929  mchbar_write32(GDCRTRAININGMOD, 0);
2930  mchbar_write32(IOSAV_DC_MASK, 0);
2931 
2932  printram("CP5b\n");
2933 
2934  FOR_ALL_POPULATED_CHANNELS {
2935  program_timings(ctrl, channel);
2936  }
2937 
2938  u32 reg, addr;
2939 
2940  /* Poll for RCOMP */
2941  while (!(mchbar_read32(RCOMP_TIMER) & (1 << 16)))
2942  ;
2943 
2944  do {
2945  reg = mchbar_read32(IOSAV_STATUS_ch(0));
2946  } while ((reg & 0x14) == 0);
2947 
2948  /* Set state of memory controller */
2949  mchbar_write32(MC_INIT_STATE_G, 0x116);
2950  mchbar_write32(MC_INIT_STATE, 0);
2951 
2952  /* Wait 500us */
2953  udelay(500);
2954 
2955  FOR_ALL_CHANNELS {
2956  /* Set valid rank CKE */
2957  reg = 0;
2958  reg = (reg & ~0x0f) | ctrl->rankmap[channel];
2959  addr = MC_INIT_STATE_ch(channel);
2960  mchbar_write32(addr, reg);
2961 
2962  /* Wait 10ns for ranks to settle */
2963  // udelay(0.01);
2964 
2965  reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
2966  mchbar_write32(addr, reg);
2967 
2968  /* Write reset using a NOP */
2969  write_reset(ctrl);
2970  }
2971 
2972  /* MRS commands */
2973  dram_mrscommands(ctrl);
2974 
2975  printram("CP5c\n");
2976 
2977  mchbar_write32(GDCRTRAININGMOD_ch(0), 0);
2978 
2979  FOR_ALL_CHANNELS {
2980  mchbar_clrbits32(GDCRCMDDEBUGMUXCFG_Cz_S(channel), 0x3f << 24);
2981  udelay(2);
2982  }
2983 }
write32
static void write32(void *addr, uint32_t val)
Definition: mmio.h:40
assert
#define assert(statement)
Definition: assert.h:74
pattern
static const u32 pattern[8]
Definition: ast_post.c:428
ARRAY_SIZE
#define ARRAY_SIZE(a)
Definition: helpers.h:12
MIN
#define MIN(a, b)
Definition: helpers.h:37
MAX
#define MAX(a, b)
Definition: helpers.h:40
DIV_ROUND_UP
#define DIV_ROUND_UP(x, y)
Definition: helpers.h:60
addr
static u32 addr
Definition: cirrus.c:14
printk
#define printk(level,...)
Definition: stdlib.h:16
die
void __noreturn die(const char *fmt,...)
Definition: die.c:17
RAM_DEBUG
#define RAM_DEBUG
Definition: console.h:13
info
static struct smmstore_params_info info
Definition: ramstage.c:12
CONFIG
@ CONFIG
Definition: dsi_common.h:201
REMAPLIMIT
#define REMAPLIMIT
Definition: e7505.h:36
REMAPBASE
#define REMAPBASE
Definition: e7505.h:35
mchbar_setbits32
#define mchbar_setbits32(addr, set)
Definition: fixed_bars.h:58
mchbar_write8
static __always_inline void mchbar_write8(const uintptr_t offset, const uint8_t value)
Definition: fixed_bars.h:26
mchbar_write32
static __always_inline void mchbar_write32(const uintptr_t offset, const uint32_t value)
Definition: fixed_bars.h:36
mchbar_clrsetbits32
static __always_inline void mchbar_clrsetbits32(uintptr_t offset, uint32_t clear, uint32_t set)
Definition: fixed_bars.h:51
mchbar_read32
static __always_inline uint32_t mchbar_read32(const uintptr_t offset)
Definition: fixed_bars.h:21
mchbar_clrbits32
#define mchbar_clrbits32(addr, clear)
Definition: fixed_bars.h:62
offset
static size_t offset
Definition: flashconsole.c:16
GGC
#define GGC
Definition: host_bridge.h:9
CAPID0_A
#define CAPID0_A
Definition: host_bridge.h:65
TOLUD
#define TOLUD
Definition: host_bridge.h:61
MESEG_BASE
#define MESEG_BASE
Definition: host_bridge.h:35
TOUUD
#define TOUUD
Definition: host_bridge.h:57
BDSM
#define BDSM
Definition: host_bridge.h:58
TOM
#define TOM
Definition: host_bridge.h:56
BGSM
#define BGSM
Definition: host_bridge.h:59
SAPMCTL
#define SAPMCTL
Definition: mchbar.h:65
MAD_DIMM
#define MAD_DIMM(ch)
Definition: mchbar.h:12
SSKPD
#define SSKPD
Definition: mchbar.h:36
NUM_SLOTS
#define NUM_SLOTS
Definition: mchbar.h:8
MC_INIT_STATE_G
#define MC_INIT_STATE_G
Definition: mchbar.h:13
MAD_CHNL
#define MAD_CHNL
Definition: mchbar.h:11
NUM_CHANNELS
#define NUM_CHANNELS
Definition: mchbar.h:7
printram
#define printram(x,...)
Convenience macro for enabling printk with CONFIG(DEBUG_RAM_SETUP)
Definition: common.h:46
TCK_1066MHZ
#define TCK_1066MHZ
Definition: common.h:21
pci_ops.h
pci_write_config32
static __always_inline void pci_write_config32(const struct device *dev, u16 reg, u32 val)
Definition: pci_ops.h:76
pci_read_config16
static __always_inline u16 pci_read_config16(const struct device *dev, u16 reg)
Definition: pci_ops.h:52
pci_read_config32
static __always_inline u32 pci_read_config32(const struct device *dev, u16 reg)
Definition: pci_ops.h:58
SPD_MEMORY_TYPE_SDRAM_DDR3
@ SPD_MEMORY_TYPE_SDRAM_DDR3
Definition: spd.h:152
BIOS_DEBUG
#define BIOS_DEBUG
BIOS_DEBUG - Verbose output.
Definition: loglevel.h:128
BIOS_EMERG
#define BIOS_EMERG
BIOS_EMERG - Emergency / Fatal.
Definition: loglevel.h:25
BIOS_ERR
#define BIOS_ERR
BIOS_ERR - System in incomplete state.
Definition: loglevel.h:72
BIOS_WARNING
#define BIOS_WARNING
BIOS_WARNING - Bad configuration.
Definition: loglevel.h:86
model_206ax.h
IS_SANDY_CPU_C
#define IS_SANDY_CPU_C(x)
Definition: model_206ax.h:26
IS_SANDY_CPU
#define IS_SANDY_CPU(x)
Definition: model_206ax.h:25
IS_SANDY_CPU_D0
#define IS_SANDY_CPU_D0(x)
Definition: model_206ax.h:27
IS_IVY_CPU_E
#define IS_IVY_CPU_E(x)
Definition: model_206ax.h:35
IS_IVY_CPU_D
#define IS_IVY_CPU_D(x)
Definition: model_206ax.h:34
IS_SANDY_CPU_D1
#define IS_SANDY_CPU_D1(x)
Definition: model_206ax.h:28
IS_IVY_CPU
#define IS_IVY_CPU(x)
Definition: model_206ax.h:31
get_platform_type
enum platform_type get_platform_type(void)
Definition: common.c:8
pci_def.h
CCC_MAX_PI
#define CCC_MAX_PI
Definition: raminit_common.c:908
lane_base
static const u32 lane_base[]
Definition: raminit_common.c:901
jedec_write_leveling
static int jedec_write_leveling(ramctr_timing *ctrl)
Definition: raminit_common.c:1786
dram_mr2
static void dram_mr2(ramctr_timing *ctrl, u8 rank, int channel)
Definition: raminit_common.c:771
dram_mr1
static void dram_mr1(ramctr_timing *ctrl, u8 rank, int channel)
Definition: raminit_common.c:762
make_mr0
static u32 make_mr0(ramctr_timing *ctrl, u8 rank)
Definition: raminit_common.c:685
normalize_training
void normalize_training(ramctr_timing *ctrl)
Definition: raminit_common.c:2532
write_level_rank
static int write_level_rank(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_common.c:1554
try_cmd_stretch
static int try_cmd_stretch(ramctr_timing *ctrl, int channel, int cmd_stretch)
Definition: raminit_common.c:1987
dram_memorymap
void dram_memorymap(ramctr_timing *ctrl, int me_uma_size)
Definition: raminit_common.c:369
does_lane_work
static int does_lane_work(ramctr_timing *ctrl, int channel, int slotrank, int lane)
Definition: raminit_common.c:1047
command_training
int command_training(ramctr_timing *ctrl)
Definition: raminit_common.c:2057
get_power_down_mode
static enum power_down_mode get_power_down_mode(ramctr_timing *ctrl)
Definition: raminit_common.c:671
find_agrsv_read_margin
static int find_agrsv_read_margin(ramctr_timing *ctrl, int channel, int slotrank, int *edges)
Definition: raminit_common.c:2264
get_host_ecc_cap
bool get_host_ecc_cap(void)
Definition: raminit_common.c:360
align_rt_io_latency
static int align_rt_io_latency(ramctr_timing *ctrl, int channel, int slotrank, int prev)
Definition: raminit_common.c:1265
aggressive_write_training
int aggressive_write_training(ramctr_timing *ctrl)
Definition: raminit_common.c:2413
set_write_vref
static void set_write_vref(const int channel, const u8 wr_vref)
Definition: raminit_common.c:2407
aggressive_read_training
int aggressive_read_training(ramctr_timing *ctrl)
Definition: raminit_common.c:2352
fill_pattern0
static void fill_pattern0(ramctr_timing *ctrl, int channel, u32 a, u32 b)
Definition: raminit_common.c:1508
tx_dq_write_leveling
static int tx_dq_write_leveling(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_common.c:1444
find_predefined_pattern
static void find_predefined_pattern(ramctr_timing *ctrl, const int channel)
Definition: raminit_common.c:2152
RCVEN_COARSE_PI_LENGTH
#define RCVEN_COARSE_PI_LENGTH
Definition: raminit_common.c:1096
dram_dimm_set_mapping
void dram_dimm_set_mapping(ramctr_timing *ctrl, int training)
Definition: raminit_common.c:290
test_rcven
static void test_rcven(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_common.c:1037
set_read_write_timings
void set_read_write_timings(ramctr_timing *ctrl)
Definition: raminit_common.c:2754
DEFAULT_PCI_MMIO_SIZE
#define DEFAULT_PCI_MMIO_SIZE
Definition: raminit_common.c:367
CT_MIN_PI
#define CT_MIN_PI
Definition: raminit_common.c:1981
dram_mrscommands
void dram_mrscommands(ramctr_timing *ctrl)
Definition: raminit_common.c:811
get_ODT
static odtmap get_ODT(ramctr_timing *ctrl, int channel)
Definition: raminit_common.c:722
get_longest_zero_run
static struct run get_longest_zero_run(int *seq, int sz)
Definition: raminit_common.c:1063
write_reset
static void write_reset(ramctr_timing *ctrl)
Definition: raminit_common.c:519
toggle_io_reset
static void toggle_io_reset(void)
Definition: raminit_common.c:27
train_write_flyby
static void train_write_flyby(ramctr_timing *ctrl)
Definition: raminit_common.c:1642
set_normal_operation
void set_normal_operation(ramctr_timing *ctrl)
Definition: raminit_common.c:2790
fill_pattern5
static void fill_pattern5(ramctr_timing *ctrl, int channel, int patno)
Definition: raminit_common.c:1933
find_rcven_pi_coarse
static void find_rcven_pi_coarse(ramctr_timing *ctrl, int channel, int slotrank, int *upperA)
Definition: raminit_common.c:1098
restore_timings
void restore_timings(ramctr_timing *ctrl)
Definition: raminit_common.c:2896
CT_PI_LENGTH
#define CT_PI_LENGTH
Definition: raminit_common.c:1983
make_mr1
static u32 make_mr1(ramctr_timing *ctrl, u8 rank, int channel)
Definition: raminit_common.c:749
get_XOVER_CMD
static u32 get_XOVER_CMD(u8 rankmap)
Definition: raminit_common.c:41
dram_find_common_params
void dram_find_common_params(ramctr_timing *ctrl)
Definition: raminit_common.c:58
program_timings
void program_timings(ramctr_timing *ctrl, int channel)
Definition: raminit_common.c:910
encode_wm
static int encode_wm(int ns)
Definition: raminit_common.c:2800
get_XOVER_CLK
static u32 get_XOVER_CLK(u8 rankmap)
Definition: raminit_common.c:36
disable_refresh_machine
static void disable_refresh_machine(ramctr_timing *ctrl)
Definition: raminit_common.c:1750
TX_DQS_PI_LENGTH
#define TX_DQS_PI_LENGTH
Definition: raminit_common.c:1552
prepare_training
void prepare_training(ramctr_timing *ctrl)
Definition: raminit_common.c:2738
final_registers
void final_registers(ramctr_timing *ctrl)
Definition: raminit_common.c:2806
CT_MAX_PI
#define CT_MAX_PI
Definition: raminit_common.c:1982
ddr3_mirror_mrreg
static void ddr3_mirror_mrreg(int *bank, u32 *addr)
Definition: raminit_common.c:595
get_precedening_channels
static int get_precedening_channels(ramctr_timing *ctrl, int target_channel)
Definition: raminit_common.c:1492
channel_scrub
void channel_scrub(ramctr_timing *ctrl)
Definition: raminit_common.c:2595
dram_xover
void dram_xover(ramctr_timing *ctrl)
Definition: raminit_common.c:101
test_command_training
static int test_command_training(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_common.c:1886
fill_pattern1
static void fill_pattern1(ramctr_timing *ctrl, int channel)
Definition: raminit_common.c:1532
find_read_mpr_margin
static int find_read_mpr_margin(ramctr_timing *ctrl, int channel, int slotrank, int *edges)
Definition: raminit_common.c:2107
encode_odt
static u32 encode_odt(u32 odt)
Definition: raminit_common.c:734
find_roundtrip_latency
static int find_roundtrip_latency(ramctr_timing *ctrl, int channel, int slotrank, int *upperA)
Definition: raminit_common.c:1179
read_mpr_training
int read_mpr_training(ramctr_timing *ctrl)
Definition: raminit_common.c:2204
get_host_ecc_forced
bool get_host_ecc_forced(void)
Definition: raminit_common.c:347
test_aggressive_write
static void test_aggressive_write(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_common.c:2398
dram_dimm_mapping
void dram_dimm_mapping(ramctr_timing *ctrl)
Definition: raminit_common.c:242
dram_mr0
static void dram_mr0(ramctr_timing *ctrl, u8 rank, int channel)
Definition: raminit_common.c:717
dram_zones
void dram_zones(ramctr_timing *ctrl, int training)
Definition: raminit_common.c:308
sfence
static void sfence(void)
Definition: raminit_common.c:21
program_wdb_pattern_length
static void program_wdb_pattern_length(int channel, const unsigned int num_cachelines)
Definition: raminit_common.c:1503
channel_test
int channel_test(ramctr_timing *ctrl)
Definition: raminit_common.c:2558
dram_odt_stretch
static void dram_odt_stretch(ramctr_timing *ctrl, int channel)
Definition: raminit_common.c:119
dram_jedecreset
void dram_jedecreset(ramctr_timing *ctrl)
Definition: raminit_common.c:539
get_logic_delay_delta
static int get_logic_delay_delta(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_common.c:1242
MIN_C320C_LEN
#define MIN_C320C_LEN
Definition: raminit_common.c:1985
fine_tune_rcven_pi
static void fine_tune_rcven_pi(ramctr_timing *ctrl, int channel, int slotrank, int *upperA)
Definition: raminit_common.c:1129
test_tx_dq
static void test_tx_dq(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_common.c:1404
tx_dq_threshold_process
static void tx_dq_threshold_process(int *data, const int count)
Definition: raminit_common.c:1425
set_wmm_behavior
void set_wmm_behavior(const u32 cpu)
Definition: raminit_common.c:2729
reprogram_320c
static void reprogram_320c(ramctr_timing *ctrl)
Definition: raminit_common.c:1968
write_mrreg
static void write_mrreg(ramctr_timing *ctrl, int channel, int slotrank, int reg, u32 val)
Definition: raminit_common.c:601
dram_timing_regs
void dram_timing_regs(ramctr_timing *ctrl)
Definition: raminit_common.c:147
num_of_channels
static int num_of_channels(const ramctr_timing *ctrl)
Definition: raminit_common.c:1524
compute_final_logic_delay
static void compute_final_logic_delay(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_common.c:1289
receive_enable_calibration
int receive_enable_calibration(ramctr_timing *ctrl)
Definition: raminit_common.c:1312
write_training
int write_training(ramctr_timing *ctrl)
Definition: raminit_common.c:1844
get_dqs_flyby_adjust
static int get_dqs_flyby_adjust(u64 val)
Definition: raminit_common.c:1622
set_scrambling_seed
void set_scrambling_seed(ramctr_timing *ctrl)
Definition: raminit_common.c:2712
dram_mr3
static void dram_mr3(ramctr_timing *ctrl, u8 rank, int channel)
Definition: raminit_common.c:806
raminit_common.h
iosav_write_prea_sequence
void iosav_write_prea_sequence(int channel, int slotrank, u32 post, u32 wrap)
Definition: raminit_iosav.c:79
MAX_TX_DQ
#define MAX_TX_DQ
Definition: raminit_common.h:429
iosav_run_queue
void iosav_run_queue(const int ch, const u8 loops, const u8 as_timer)
Definition: raminit_iosav.c:28
IOSAV_ACT
#define IOSAV_ACT
Definition: raminit_common.h:20
iosav_write_aggressive_write_read_sequence
void iosav_write_aggressive_write_read_sequence(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_iosav.c:681
MAX_CAS
#define MAX_CAS
Definition: raminit_common.h:432
iosav_write_command_training_sequence
void iosav_write_command_training_sequence(ramctr_timing *ctrl, int channel, int slotrank, unsigned int address)
Definition: raminit_iosav.c:483
FOR_ALL_LANES
#define FOR_ALL_LANES
Definition: raminit_common.h:424
power_down_mode
power_down_mode
Definition: raminit_common.h:309
PDM_NONE
@ PDM_NONE
Definition: raminit_common.h:310
PDM_PPD
@ PDM_PPD
Definition: raminit_common.h:312
PDM_DLL_OFF
@ PDM_DLL_OFF
Definition: raminit_common.h:314
PDM_APD_DLL_OFF
@ PDM_APD_DLL_OFF
Definition: raminit_common.h:315
PDM_APD_PPD
@ PDM_APD_PPD
Definition: raminit_common.h:313
iosav_write_memory_test_sequence
void iosav_write_memory_test_sequence(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_iosav.c:776
IOSAV_RD
#define IOSAV_RD
Definition: raminit_common.h:21
SSQ_WR
#define SSQ_WR
Definition: raminit_common.h:29
NUM_SLOTRANKS
#define NUM_SLOTRANKS
Definition: raminit_common.h:12
MAKE_ERR
#define MAKE_ERR
Definition: raminit_common.h:442
IOSAV_NOP
#define IOSAV_NOP
Definition: raminit_common.h:24
iosav_write_read_mpr_sequence
void iosav_write_read_mpr_sequence(int channel, int slotrank, u32 tMOD, u32 loops, u32 gap, u32 loops2, u32 post2)
Definition: raminit_iosav.c:108
iosav_write_zqcs_sequence
void iosav_write_zqcs_sequence(int channel, int slotrank, u32 gap, u32 post, u32 wrap)
Definition: raminit_iosav.c:51
NUM_LANES
#define NUM_LANES
Definition: raminit_common.h:14
iosav_write_misc_write_sequence
void iosav_write_misc_write_sequence(ramctr_timing *ctrl, int channel, int slotrank, u32 gap0, u32 loops0, u32 gap1, u32 loops2, u32 wrap2)
Definition: raminit_iosav.c:385
SSQ_NA
#define SSQ_NA
Definition: raminit_common.h:27
iosav_run_once_and_wait
void iosav_run_once_and_wait(const int ch)
Definition: raminit_iosav.c:45
IOSAV_MRS
#define IOSAV_MRS
Definition: raminit_common.h:17
NUM_PATTERNS
#define NUM_PATTERNS
Definition: raminit_common.h:302
iosav_write_prea_act_read_sequence
void iosav_write_prea_act_read_sequence(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_iosav.c:201
IOSAV_WR
#define IOSAV_WR
Definition: raminit_common.h:23
wait_for_iosav
void wait_for_iosav(int channel)
Definition: raminit_iosav.c:37
iosav_write_sequence
void iosav_write_sequence(const int ch, const struct iosav_ssq *seq, const unsigned int length)
Definition: raminit_iosav.c:16
QCLK_PI
#define QCLK_PI
Definition: raminit_common.h:440
FOR_ALL_POPULATED_RANKS
#define FOR_ALL_POPULATED_RANKS
Definition: raminit_common.h:426
iosav_write_data_write_sequence
void iosav_write_data_write_sequence(ramctr_timing *ctrl, int channel, int slotrank)
Definition: raminit_iosav.c:586
tDLLK
#define tDLLK
Definition: raminit_common.h:9
SSQ_RD
#define SSQ_RD
Definition: raminit_common.h:28
FOR_ALL_POPULATED_CHANNELS
#define FOR_ALL_POPULATED_CHANNELS
Definition: raminit_common.h:427
iosav_write_jedec_write_leveling_sequence
void iosav_write_jedec_write_leveling_sequence(ramctr_timing *ctrl, int channel, int slotrank, int bank, u32 mr1reg)
Definition: raminit_iosav.c:298
IOSAV_ZQCS
#define IOSAV_ZQCS
Definition: raminit_common.h:19
MAX_EDGE_TIMING
#define MAX_EDGE_TIMING
Definition: raminit_common.h:428
FOR_ALL_CHANNELS
#define FOR_ALL_CHANNELS
Definition: raminit_common.h:425
MIN_CAS
#define MIN_CAS
Definition: raminit_common.h:433
IOSAV_PRE
#define IOSAV_PRE
Definition: raminit_common.h:18
raminit_native.h
use_base
const u8 use_base[63][32]
Definition: raminit_tables.c:225
invert
const u8 invert[63][32]
Definition: raminit_tables.c:480
raminit_tables.h
ME_STLEN_EN
#define ME_STLEN_EN
Definition: host_bridge.h:29
CAPID_WRTVREF
#define CAPID_WRTVREF
Definition: host_bridge.h:55
MESEG_MASK
#define MESEG_MASK
Definition: host_bridge.h:27
TSEGMB
#define TSEGMB
Definition: host_bridge.h:48
MELCK
#define MELCK
Definition: host_bridge.h:28
WMM_READ_CONFIG
#define WMM_READ_CONFIG
Opportunistic reads configuration during write-major-mode (WMM)
Definition: mchbar.h:429
GDCRCKLOGICDELAY_ch
#define GDCRCKLOGICDELAY_ch(ch)
Definition: mchbar.h:299
PM_CMD_PWR_ch
#define PM_CMD_PWR_ch(ch)
Definition: mchbar.h:392
SCRAMBLING_SEED_2_LO_ch
#define SCRAMBLING_SEED_2_LO_ch(ch)
Definition: mchbar.h:341
MEM_TRML_ESTIMATION_CONFIG
#define MEM_TRML_ESTIMATION_CONFIG
Definition: mchbar.h:497
MC_INIT_STATE_ch
#define MC_INIT_STATE_ch(ch)
Definition: mchbar.h:379
GDCRCKPICODE_ch
#define GDCRCKPICODE_ch(ch)
Definition: mchbar.h:298
GDCRRX
#define GDCRRX(ch, rank)
Definition: mchbar.h:292
SCHED_SECOND_CBIT_ch
#define SCHED_SECOND_CBIT_ch(ch)
Definition: mchbar.h:336
LANEBASE_B3
#define LANEBASE_B3
Definition: mchbar.h:281
GDCRTRAININGRESULT
#define GDCRTRAININGRESULT(ch, y)
Definition: mchbar.h:289
BANDTIMERS_IVB
#define BANDTIMERS_IVB
Definition: mchbar.h:521
MEM_TRML_INTERRUPT
#define MEM_TRML_INTERRUPT
Definition: mchbar.h:499
IOSAV_BYTE_SERROR_C_ch
#define IOSAV_BYTE_SERROR_C_ch(ch)
Definition: mchbar.h:389
LANEBASE_B1
#define LANEBASE_B1
Definition: mchbar.h:279
PM_DLL_CONFIG
#define PM_DLL_CONFIG
Definition: mchbar.h:480
LANEBASE_ECC
#define LANEBASE_ECC
Definition: mchbar.h:282
SAPMTIMERS
#define SAPMTIMERS
Definition: mchbar.h:514
GDCRTRAININGMOD_ch
#define GDCRTRAININGMOD_ch(ch)
Definition: mchbar.h:307
TC_RFP_ch
#define TC_RFP_ch(ch)
Definition: mchbar.h:376
PM_PDWN_CONFIG
#define PM_PDWN_CONFIG
Definition: mchbar.h:423
TC_DBP_ch
#define TC_DBP_ch(ch)
Definition: mchbar.h:328
IOSAV_By_ERROR_COUNT
#define IOSAV_By_ERROR_COUNT(y)
Definition: mchbar.h:453
IOSAV_DC_MASK
#define IOSAV_DC_MASK
Definition: mchbar.h:451
SC_WDBWM
#define SC_WDBWM
Definition: mchbar.h:463
TC_RFTP_ch
#define TC_RFTP_ch(ch)
Definition: mchbar.h:377
GDCRCMDDEBUGMUXCFG_Cz_S
#define GDCRCMDDEBUGMUXCFG_Cz_S(ch)
Definition: mchbar.h:303
TC_ZQCAL_ch
#define TC_ZQCAL_ch(ch)
Definition: mchbar.h:375
MAD_ZR
#define MAD_ZR
Definition: mchbar.h:472
MCMNTS_SPARE
#define MCMNTS_SPARE
Auxiliary register in mcmnts synthesis FUB (Functional Unit Block).
Definition: mchbar.h:448
GDCRTRAININGRESULT2
#define GDCRTRAININGRESULT2(ch)
Definition: mchbar.h:291
IOSAV_STATUS_ch
#define IOSAV_STATUS_ch(ch)
Definition: mchbar.h:374
IOSAV_By_BW_SERROR_C
#define IOSAV_By_BW_SERROR_C(y)
Definition: mchbar.h:431
SC_IO_LATENCY_ch
#define SC_IO_LATENCY_ch(ch)
Definition: mchbar.h:339
TC_RWP_ch
#define TC_RWP_ch(ch)
Definition: mchbar.h:330
PM_BW_LIMIT_CONFIG
#define PM_BW_LIMIT_CONFIG
Definition: mchbar.h:462
LANEBASE_B6
#define LANEBASE_B6
Definition: mchbar.h:285
LANEBASE_B2
#define LANEBASE_B2
Definition: mchbar.h:280
IOSAV_n_ADDRESS_LFSR_ch
#define IOSAV_n_ADDRESS_LFSR_ch(ch, y)
Definition: mchbar.h:369
LANEBASE_B5
#define LANEBASE_B5
Definition: mchbar.h:284
IOSAV_By_BW_MASK_ch
#define IOSAV_By_BW_MASK_ch(ch, y)
Definition: mchbar.h:348
LANEBASE_B0
#define LANEBASE_B0
Definition: mchbar.h:278
IOSAV_By_BW_SERROR_C_ch
#define IOSAV_By_BW_SERROR_C_ch(ch, y)
Definition: mchbar.h:362
SCRAMBLING_SEED_1_ch
#define SCRAMBLING_SEED_1_ch(ch)
Definition: mchbar.h:340
RCOMP_TIMER
#define RCOMP_TIMER
Definition: mchbar.h:481
BANDTIMERS_SNB
#define BANDTIMERS_SNB
Definition: mchbar.h:517
LANEBASE_B4
#define LANEBASE_B4
Definition: mchbar.h:283
SCHED_CBIT_ch
#define SCHED_CBIT_ch(ch)
Definition: mchbar.h:337
SCRAMBLING_SEED_2_HI_ch
#define SCRAMBLING_SEED_2_HI_ch(ch)
Definition: mchbar.h:342
MEM_TRML_THRESHOLDS_CONFIG
#define MEM_TRML_THRESHOLDS_CONFIG
Definition: mchbar.h:498
BROADCAST_CH
#define BROADCAST_CH
Definition: mchbar.h:325
TC_OTHP_ch
#define TC_OTHP_ch(ch)
Definition: mchbar.h:331
SC_ROUNDT_LAT_ch
#define SC_ROUNDT_LAT_ch(ch)
Definition: mchbar.h:338
LANEBASE_B7
#define LANEBASE_B7
Definition: mchbar.h:286
TC_MR2_SHADOW_ch
#define TC_MR2_SHADOW_ch(ch)
Definition: mchbar.h:378
SAPMTIMERS2_IVB
#define SAPMTIMERS2_IVB
WARNING: Only applies to Ivy Bridge!
Definition: mchbar.h:520
MC_INIT_STATE
#define MC_INIT_STATE
Definition: mchbar.h:441
GDCRTRAININGMOD
#define GDCRTRAININGMOD
Definition: mchbar.h:316
TC_DTP_ch
#define TC_DTP_ch(ch)
WARNING: Only applies to Ivy Bridge!
Definition: mchbar.h:334
PM_TRML_M_CONFIG_ch
#define PM_TRML_M_CONFIG_ch(ch)
Definition: mchbar.h:391
GDCRCMDPICODING_ch
#define GDCRCMDPICODING_ch(ch)
Definition: mchbar.h:314
GDCRTX
#define GDCRTX(ch, rank)
Definition: mchbar.h:293
TC_SRFTP_ch
#define TC_SRFTP_ch(ch)
Definition: mchbar.h:380
GDCRTRAININGRESULT1
#define GDCRTRAININGRESULT1(ch)
Definition: mchbar.h:290
TC_RAP_ch
#define TC_RAP_ch(ch)
Definition: mchbar.h:329
IOSAV_DATA_CTL_ch
#define IOSAV_DATA_CTL_ch(ch)
Definition: mchbar.h:373
IOSAV_By_ERROR_COUNT_ch
#define IOSAV_By_ERROR_COUNT_ch(ch, y)
Definition: mchbar.h:384
IOSAV_By_BW_SERROR_ch
#define IOSAV_By_BW_SERROR_ch(ch, y)
Definition: mchbar.h:345
PLATFORM_MOBILE
@ PLATFORM_MOBILE
Definition: sandybridge.h:19
base
uintptr_t base
Definition: uart.c:17
HOST_BRIDGE
@ HOST_BRIDGE
Definition: reg_access.h:23
ecc
u32 ecc
Definition: report_platform.c:15
uint16_t
unsigned short uint16_t
Definition: stdint.h:11
u64
uint64_t u64
Definition: stdint.h:54
u32
uint32_t u32
Definition: stdint.h:51
uintptr_t
unsigned long uintptr_t
Definition: stdint.h:21
u16
uint16_t u16
Definition: stdint.h:48
u8
uint8_t u8
Definition: stdint.h:45
dimm_attr_ddr3_st
DIMM characteristics.
Definition: ddr3.h:106
dimm_attr_ddr3_st::cas_supported
u16 cas_supported
Definition: ddr3.h:109
dimm_attr_ddr3_st::dram_type
enum spd_memory_type dram_type
Definition: ddr3.h:107
dimm_attr_ddr3_st::tWTR
u32 tWTR
Definition: ddr3.h:132
dimm_attr_ddr3_st::tRCD
u32 tRCD
Definition: ddr3.h:126
dimm_attr_ddr3_st::tCWL
u32 tCWL
Definition: ddr3.h:135
dimm_attr_ddr3_st::tRAS
u32 tRAS
Definition: ddr3.h:129
dimm_attr_ddr3_st::tRRD
u32 tRRD
Definition: ddr3.h:127
dimm_attr_ddr3_st::tCK
u32 tCK
Definition: ddr3.h:123
dimm_attr_ddr3_st::tAA
u32 tAA
Definition: ddr3.h:124
dimm_attr_ddr3_st::size_mb
u32 size_mb
Definition: ddr3.h:121
dimm_attr_ddr3_st::tRP
u32 tRP
Definition: ddr3.h:128
dimm_attr_ddr3_st::tRTP
u32 tRTP
Definition: ddr3.h:133
dimm_attr_ddr3_st::width
u8 width
Definition: ddr3.h:113
dimm_attr_ddr3_st::tRFC
u32 tRFC
Definition: ddr3.h:131
dimm_attr_ddr3_st::tFAW
u32 tFAW
Definition: ddr3.h:134
dimm_attr_ddr3_st::tCMD
u16 tCMD
Definition: ddr3.h:136
dimm_attr_ddr3_st::ranks
u8 ranks
Definition: ddr3.h:115
dimm_attr_ddr3_st::tWR
u32 tWR
Definition: ddr3.h:125
dimm_info
If this table is filled and put in CBMEM, then these info in CBMEM will be used to generate smbios ty...
Definition: memory_info.h:19
iosav_ssq
Definition: raminit_common.h:32
iosav_ssq::rank
u32 rank
Definition: raminit_common.h:66
iosav_ssq::sp_cmd_ctrl
union iosav_ssq::@334 sp_cmd_ctrl
iosav_ssq::bank
u32 bank
Definition: raminit_common.h:64
iosav_ssq::command
u32 command
Definition: raminit_common.h:36
odtmap_st
Definition: raminit_common.h:318
odtmap_st::rttnom
u16 rttnom
Definition: raminit_common.h:320
odtmap_st::rttwr
u16 rttwr
Definition: raminit_common.h:319
ram_rank_timings::ram_lane_timings::rx_dqs_n
u8 rx_dqs_n
Definition: raminit_common.h:343
ram_rank_timings::ram_lane_timings::rcven
u16 rcven
Definition: raminit_common.h:341
ram_rank_timings::ram_lane_timings::tx_dqs
u16 tx_dqs
Definition: raminit_common.h:347
ram_rank_timings::ram_lane_timings::rx_dqs_p
u8 rx_dqs_p
Definition: raminit_common.h:342
ram_rank_timings::ram_lane_timings::tx_dq
int tx_dq
Definition: raminit_common.h:346
ram_rank_timings
Definition: raminit_common.h:329
ram_rank_timings::lanes
struct ram_rank_timings::ram_lane_timings lanes[NUM_LANES]
ram_rank_timings::pi_coding
int pi_coding
Definition: raminit_common.h:337
ram_rank_timings::io_latency
u8 io_latency
Definition: raminit_common.h:334
ram_rank_timings::roundtrip_latency
u8 roundtrip_latency
Definition: raminit_common.h:331
ramctr_timing_st
Definition: raminit_common.h:352
ramctr_timing_st::tRTP
u32 tRTP
Definition: raminit_common.h:375
ramctr_timing_st::tCKE
u32 tCKE
Definition: raminit_common.h:388
ramctr_timing_st::extended_temperature_range
int extended_temperature_range
Definition: raminit_common.h:412
ramctr_timing_st::mdll_wake_delay
u16 mdll_wake_delay
Definition: raminit_common.h:394
ramctr_timing_st::pi_code_offset
int pi_code_offset
Definition: raminit_common.h:402
ramctr_timing_st::info
dimm_info info
Definition: raminit_common.h:419
ramctr_timing_st::tCK
u32 tCK
Definition: raminit_common.h:366
ramctr_timing_st::cas_supported
u16 cas_supported
Definition: raminit_common.h:364
ramctr_timing_st::tAA
u32 tAA
Definition: raminit_common.h:367
ramctr_timing_st::ecc_enabled
bool ecc_enabled
Definition: raminit_common.h:407
ramctr_timing_st::tRRD
u32 tRRD
Definition: raminit_common.h:370
ramctr_timing_st::tXP
u32 tXP
Definition: raminit_common.h:390
ramctr_timing_st::tWR
u32 tWR
Definition: raminit_common.h:368
ramctr_timing_st::tRFC
u32 tRFC
Definition: raminit_common.h:373
ramctr_timing_st::tx_dq_offset
int tx_dq_offset[3]
Definition: raminit_common.h:410
ramctr_timing_st::tRCD
u32 tRCD
Definition: raminit_common.h:369
ramctr_timing_st::tMOD
u32 tMOD
Definition: raminit_common.h:385
ramctr_timing_st::rankmap
u8 rankmap[NUM_CHANNELS]
Definition: raminit_common.h:396
ramctr_timing_st::pi_coding_threshold
int pi_coding_threshold
Definition: raminit_common.h:403
ramctr_timing_st::cpu
u32 cpu
Definition: raminit_common.h:356
ramctr_timing_st::CWL
u8 CWL
Definition: raminit_common.h:382
ramctr_timing_st::tRP
u32 tRP
Definition: raminit_common.h:371
ramctr_timing_st::tFAW
u32 tFAW
Definition: raminit_common.h:376
ramctr_timing_st::rank_mirror
int rank_mirror[NUM_CHANNELS][NUM_SLOTRANKS]
Definition: raminit_common.h:415
ramctr_timing_st::tCMD
u32 tCMD
Definition: raminit_common.h:378
ramctr_timing_st::tAONPD
u32 tAONPD
Definition: raminit_common.h:391
ramctr_timing_st::tCWL
u32 tCWL
Definition: raminit_common.h:377
ramctr_timing_st::cmd_stretch
u32 cmd_stretch[NUM_CHANNELS]
Definition: raminit_common.h:400
ramctr_timing_st::tXSOffset
u32 tXSOffset
Definition: raminit_common.h:386
ramctr_timing_st::ref_card_offset
int ref_card_offset[NUM_CHANNELS]
Definition: raminit_common.h:397
ramctr_timing_st::channel_size_mb
int channel_size_mb[NUM_CHANNELS]
Definition: raminit_common.h:399
ramctr_timing_st::lanes
int lanes
Definition: raminit_common.h:408
ramctr_timing_st::mad_dimm
u32 mad_dimm[NUM_CHANNELS]
Definition: raminit_common.h:398
ramctr_timing_st::auto_self_refresh
int auto_self_refresh
Definition: raminit_common.h:413
ramctr_timing_st::timings
struct ram_rank_timings timings[NUM_CHANNELS][NUM_SLOTRANKS]
Definition: raminit_common.h:417
ramctr_timing_st::edge_offset
int edge_offset[3]
Definition: raminit_common.h:409
ramctr_timing_st::tREFI
u32 tREFI
Definition: raminit_common.h:384
ramctr_timing_st::tWTR
u32 tWTR
Definition: raminit_common.h:374
ramctr_timing_st::tRAS
u32 tRAS
Definition: raminit_common.h:372
ramctr_timing_st::CAS
u8 CAS
Definition: raminit_common.h:381
ramctr_timing_st::tXPDLL
u32 tXPDLL
Definition: raminit_common.h:389
run::start
int start
Definition: raminit_common.c:1058
run::length
int length
Definition: raminit_common.c:1060
run::middle
int middle
Definition: raminit_common.c:1056
val
u8 val
Definition: sys.c:300
udelay
void udelay(uint32_t us)
Definition: udelay.c:15
gdcr_cmd_pi_coding_reg
Definition: raminit_common.h:117
gdcr_cmd_pi_coding_reg::ctl_logic_delay_d1
u32 ctl_logic_delay_d1
Definition: raminit_common.h:128
gdcr_cmd_pi_coding_reg::cmd_pi_code
u32 cmd_pi_code
Definition: raminit_common.h:119
gdcr_cmd_pi_coding_reg::ctl_logic_delay_d0
u32 ctl_logic_delay_d0
Definition: raminit_common.h:124
gdcr_cmd_pi_coding_reg::cmd_logic_delay
u32 cmd_logic_delay
Definition: raminit_common.h:121
gdcr_cmd_pi_coding_reg::ctl_pi_code_d1
u32 ctl_pi_code_d1
Definition: raminit_common.h:127
gdcr_cmd_pi_coding_reg::ctl_pi_code_d0
u32 ctl_pi_code_d0
Definition: raminit_common.h:120
gdcr_cmd_pi_coding_reg::raw
u32 raw
Definition: raminit_common.h:133
gdcr_rx_reg
Definition: raminit_common.h:89
gdcr_rx_reg::rcven_pi_code
u32 rcven_pi_code
Definition: raminit_common.h:91
gdcr_rx_reg::raw
u32 raw
Definition: raminit_common.h:100
gdcr_training_mod_reg
Definition: raminit_common.h:136
gdcr_training_mod_reg::dq_dqs_training_res
u32 dq_dqs_training_res
Definition: raminit_common.h:143
gdcr_training_mod_reg::receive_enable_mode
u32 receive_enable_mode
Definition: raminit_common.h:138
gdcr_training_mod_reg::write_leveling_mode
u32 write_leveling_mode
Definition: raminit_common.h:139
gdcr_training_mod_reg::vref_gen_ctl
u32 vref_gen_ctl
Definition: raminit_common.h:152
gdcr_training_mod_reg::raw
u32 raw
Definition: raminit_common.h:156
gdcr_tx_reg
Definition: raminit_common.h:103
gdcr_tx_reg::raw
u32 raw
Definition: raminit_common.h:114
gdcr_tx_reg::tx_dq_pi_code
u32 tx_dq_pi_code
Definition: raminit_common.h:105
tc_dbp_reg
Definition: raminit_common.h:176
tc_dbp_reg::raw
u32 raw
Definition: raminit_common.h:185
tc_dbp_reg::tRCD
u32 tRCD
Definition: raminit_common.h:178
tc_dtp_reg
Definition: raminit_common.h:237
tc_dtp_reg::raw
u32 raw
Definition: raminit_common.h:244
tc_dtp_reg::overclock_tXP
u32 overclock_tXP
Definition: raminit_common.h:240
tc_othp_reg
Definition: raminit_common.h:223
tc_othp_reg::odt_delay_d0
u32 odt_delay_d0
Definition: raminit_common.h:230
tc_othp_reg::odt_delay_d1
u32 odt_delay_d1
Definition: raminit_common.h:231
tc_othp_reg::tCPDED
u32 tCPDED
Definition: raminit_common.h:228
tc_othp_reg::tXPDLL
u32 tXPDLL
Definition: raminit_common.h:225
tc_othp_reg::raw
u32 raw
Definition: raminit_common.h:234
tc_rap_reg
Definition: raminit_common.h:188
tc_rap_reg::raw
u32 raw
Definition: raminit_common.h:199
tc_rap_reg::tRRD
u32 tRRD
Definition: raminit_common.h:190
tc_rfp_reg
Definition: raminit_common.h:247
tc_rfp_reg::refresh_2x_control
u32 refresh_2x_control
Definition: raminit_common.h:252
tc_rfp_reg::raw
u32 raw
Definition: raminit_common.h:255
tc_rfp_reg::oref_ri
u32 oref_ri
Definition: raminit_common.h:249
tc_rftp_reg
Definition: raminit_common.h:258
tc_rftp_reg::raw
u32 raw
Definition: raminit_common.h:264
tc_rftp_reg::tREFI
u32 tREFI
Definition: raminit_common.h:260
tc_rwp_reg
Definition: raminit_common.h:202
tc_rwp_reg::tRRDR
u32 tRRDR
Definition: raminit_common.h:204
tc_rwp_reg::raw
u32 raw
Definition: raminit_common.h:220
tc_rwp_reg::tWRDRDD
u32 tWRDRDD
Definition: raminit_common.h:214
tc_srftp_reg
Definition: raminit_common.h:267
tc_srftp_reg::tXSDLL
u32 tXSDLL
Definition: raminit_common.h:269
tc_srftp_reg::raw
u32 raw
Definition: raminit_common.h:275
count
#define count
rcven
void rcven(struct sysinfo *s)
Definition: rcven.c:282