coreboot
coreboot is an Open Source project aimed at replacing the proprietary BIOS found in most computers.
raminit.c
Go to the documentation of this file.
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 
3 #include <console/console.h>
4 #include <commonlib/helpers.h>
5 #include <string.h>
6 #include <arch/io.h>
7 #include <device/mmio.h>
8 #include <device/pci_ops.h>
9 #include <device/smbus_host.h>
10 #include <cpu/x86/msr.h>
11 #include <cpu/x86/cache.h>
12 #include <cbmem.h>
13 #include <cf9_reset.h>
14 #include <ip_checksum.h>
15 #include <option.h>
16 #include <device/pci_def.h>
17 #include <device/device.h>
18 #include <halt.h>
19 #include <spd.h>
20 #include <timestamp.h>
21 #include <cpu/x86/mtrr.h>
22 #include <cpu/intel/speedstep.h>
23 #include <cpu/intel/turbo.h>
24 #include <mrc_cache.h>
25 #include <southbridge/intel/ibexpeak/me.h>
26 #include <southbridge/intel/common/pmbase.h>
27 #include <delay.h>
28 #include <types.h>
29 
30 #include "chip.h"
31 #include "ironlake.h"
32 #include "raminit.h"
33 #include "raminit_tables.h"
34 
35 #define NORTHBRIDGE PCI_DEV(0, 0, 0)
36 #define SOUTHBRIDGE PCI_DEV(0, 0x1f, 0)
37 #define GMA PCI_DEV (0, 0x2, 0x0)
38 
39 #define FOR_ALL_RANKS \
40  for (channel = 0; channel < NUM_CHANNELS; channel++) \
41  for (slot = 0; slot < NUM_SLOTS; slot++) \
42  for (rank = 0; rank < NUM_RANKS; rank++)
43 
44 #define FOR_POPULATED_RANKS \
45  for (channel = 0; channel < NUM_CHANNELS; channel++) \
46  for (slot = 0; slot < NUM_SLOTS; slot++) \
47  for (rank = 0; rank < NUM_RANKS; rank++) \
48  if (info->populated_ranks[channel][slot][rank])
49 
50 #define FOR_POPULATED_RANKS_BACKWARDS \
51  for (channel = NUM_CHANNELS - 1; channel >= 0; channel--) \
52  for (slot = 0; slot < NUM_SLOTS; slot++) \
53  for (rank = 0; rank < NUM_RANKS; rank++) \
54  if (info->populated_ranks[channel][slot][rank])
55 
56 #include <lib.h> /* Prototypes */
57 
58 typedef struct _u128 {
59  u64 lo;
60  u64 hi;
61 } u128;
62 
63 static void read128(u32 addr, u64 * out)
64 {
65  u128 ret;
66  u128 stor;
67  asm volatile ("movdqu %%xmm0, %0\n"
68  "movdqa (%2), %%xmm0\n"
69  "movdqu %%xmm0, %1\n"
70  "movdqu %0, %%xmm0":"+m" (stor), "=m"(ret):"r"(addr));
71  out[0] = ret.lo;
72  out[1] = ret.hi;
73 }
74 
75 /*
76  * Ironlake memory I/O timings are located in scan chains, accessible
77  * through MCHBAR register groups. Each channel has a scan chain, and
78  * there's a global scan chain too. Each chain is broken into smaller
79  * sections of N bits, where N <= 32. Each section allows reading and
80  * writing a certain parameter. Each section contains N - 2 data bits
81  * and two additional bits: a Mask bit, and a Halt bit.
82  */
83 
84 /* OK */
85 static void write_1d0(u32 val, u16 addr, int bits, int flag)
86 {
87  mchbar_write32(0x1d0, 0);
88  while (mchbar_read32(0x1d0) & (1 << 23))
89  ;
90  mchbar_write32(0x1d4, (val & ((1 << bits) - 1)) | 2 << bits | flag << bits);
91  mchbar_write32(0x1d0, 1 << 30 | addr);
92  while (mchbar_read32(0x1d0) & (1 << 23))
93  ;
94 }
95 
96 /* OK */
97 static u16 read_1d0(u16 addr, int split)
98 {
99  u32 val;
100  mchbar_write32(0x1d0, 0);
101  while (mchbar_read32(0x1d0) & (1 << 23))
102  ;
103  mchbar_write32(0x1d0, 1 << 31 | (((mchbar_read8(0x246) >> 2) & 3) + 0x361 - addr));
104  while (mchbar_read32(0x1d0) & (1 << 23))
105  ;
106  val = mchbar_read32(0x1d8);
107  write_1d0(0, 0x33d, 0, 0);
108  write_1d0(0, 0x33d, 0, 0);
109  val &= ((1 << split) - 1);
110  // printk (BIOS_ERR, "R1D0C [%x] => %x\n", addr, val);
111  return val;
112 }
113 
114 static void sfence(void)
115 {
116  asm volatile ("sfence");
117 }
118 
119 static inline u16 get_lane_offset(int slot, int rank, int lane)
120 {
121  return 0x124 * lane + ((lane & 4) ? 0x23e : 0) + 11 * rank + 22 * slot -
122  0x452 * (lane == 8);
123 }
124 
125 static inline u16 get_timing_register_addr(int lane, int tm, int slot, int rank)
126 {
127  const u16 offs[] = { 0x1d, 0xa8, 0xe6, 0x5c };
128  return get_lane_offset(slot, rank, lane) + offs[(tm + 3) % 4];
129 }
130 
131 static u32 gav_real(int line, u32 in)
132 {
133  // printk (BIOS_DEBUG, "%d: GAV: %x\n", line, in);
134  return in;
135 }
136 
137 #define gav(x) gav_real (__LINE__, (x))
138 
139 /* Global allocation of timings_car */
140 timing_bounds_t timings_car[64];
141 
142 /* OK */
143 static u16
144 read_500(struct raminfo *info, int channel, u16 addr, int split)
145 {
146  u32 val;
147  info->last_500_command[channel] = 1 << 31;
148  mchbar_write32(0x500 + (channel << 10), 0);
149  while (mchbar_read32(0x500 + (channel << 10)) & (1 << 23))
150  ;
151  mchbar_write32(0x500 + (channel << 10),
152  1 << 31 | (((mchbar_read8(0x246 + (channel << 10)) >> 2) & 3) + 0xb88 - addr));
153  while (mchbar_read32(0x500 + (channel << 10)) & (1 << 23))
154  ;
155  val = mchbar_read32(0x508 + (channel << 10));
156  return val & ((1 << split) - 1);
157 }
158 
159 /* OK */
160 static void
161 write_500(struct raminfo *info, int channel, u32 val, u16 addr, int bits,
162  int flag)
163 {
164  if (info->last_500_command[channel] == 1 << 31) {
165  info->last_500_command[channel] = 1 << 30;
166  write_500(info, channel, 0, 0xb61, 0, 0);
167  }
168  mchbar_write32(0x500 + (channel << 10), 0);
169  while (mchbar_read32(0x500 + (channel << 10)) & (1 << 23))
170  ;
171  mchbar_write32(0x504 + (channel << 10),
172  (val & ((1 << bits) - 1)) | 2 << bits | flag << bits);
173  mchbar_write32(0x500 + (channel << 10), 1 << 30 | addr);
174  while (mchbar_read32(0x500 + (channel << 10)) & (1 << 23))
175  ;
176 }
177 
178 static void rmw_500(struct raminfo *info, int channel, u16 addr, int bits, u32 and, u32 or)
179 {
180  const u32 val = read_500(info, channel, addr, bits) & and;
181  write_500(info, channel, val | or, addr, bits, 1);
182 }
183 
184 static int rw_test(int rank)
185 {
186  const u32 mask = 0xf00fc33c;
187  int ok = 0xff;
188  int i;
189  for (i = 0; i < 64; i++)
190  write32p((rank << 28) | (i << 2), 0);
191  sfence();
192  for (i = 0; i < 64; i++)
193  gav(read32p((rank << 28) | (i << 2)));
194  sfence();
195  for (i = 0; i < 32; i++) {
196  u32 pat = (((mask >> i) & 1) ? 0xffffffff : 0);
197  write32p((rank << 28) | (i << 3), pat);
198  write32p((rank << 28) | (i << 3) | 4, pat);
199  }
200  sfence();
201  for (i = 0; i < 32; i++) {
202  u8 pat = (((mask >> i) & 1) ? 0xff : 0);
203  int j;
204  u32 val;
205  gav(val = read32p((rank << 28) | (i << 3)));
206  for (j = 0; j < 4; j++)
207  if (((val >> (j * 8)) & 0xff) != pat)
208  ok &= ~(1 << j);
209  gav(val = read32p((rank << 28) | (i << 3) | 4));
210  for (j = 0; j < 4; j++)
211  if (((val >> (j * 8)) & 0xff) != pat)
212  ok &= ~(16 << j);
213  }
214  sfence();
215  for (i = 0; i < 64; i++)
216  write32p((rank << 28) | (i << 2), 0);
217  sfence();
218  for (i = 0; i < 64; i++)
219  gav(read32p((rank << 28) | (i << 2)));
220 
221  return ok;
222 }
223 
224 static void
225 program_timings(struct raminfo *info, u16 base, int channel, int slot, int rank)
226 {
227  int lane;
228  for (lane = 0; lane < 8; lane++) {
229  write_500(info, channel,
230  base +
231  info->training.
232  lane_timings[2][channel][slot][rank][lane],
233  get_timing_register_addr(lane, 2, slot, rank), 9, 0);
234  write_500(info, channel,
235  base +
236  info->training.
237  lane_timings[3][channel][slot][rank][lane],
238  get_timing_register_addr(lane, 3, slot, rank), 9, 0);
239  }
240 }
241 
242 static void write_26c(int channel, u16 si)
243 {
244  mchbar_write32(0x26c + (channel << 10), 0x03243f35);
245  mchbar_write32(0x268 + (channel << 10), 0xcfc00000 | si << 9);
246  mchbar_write16(0x2b9 + (channel << 10), si);
247 }
248 
249 static void toggle_1d0_142_5ff(void)
250 {
251  u32 reg32 = gav(read_1d0(0x142, 3));
252  if (reg32 & (1 << 1))
253  write_1d0(0, 0x142, 3, 1);
254 
255  mchbar_write8(0x5ff, 0);
256  mchbar_write8(0x5ff, 1 << 7);
257  if (reg32 & (1 << 1))
258  write_1d0(0x2, 0x142, 3, 1);
259 }
260 
261 static u32 get_580(int channel, u8 addr)
262 {
263  u32 ret;
264  toggle_1d0_142_5ff();
265  mchbar_write32(0x580 + (channel << 10), 0x8493c012 | addr);
266  mchbar_setbits8(0x580 + (channel << 10), 1 << 0);
267  while (!((ret = mchbar_read32(0x580 + (channel << 10))) & (1 << 16)))
268  ;
269  mchbar_clrbits8(0x580 + (channel << 10), 1 << 0);
270  return ret;
271 }
272 
273 #define RANK_SHIFT 28
274 #define CHANNEL_SHIFT 10
275 
276 static void seq9(struct raminfo *info, int channel, int slot, int rank)
277 {
278  int i, lane;
279 
280  for (i = 0; i < 2; i++)
281  for (lane = 0; lane < 8; lane++)
282  write_500(info, channel,
283  info->training.lane_timings[i +
284  1][channel][slot]
285  [rank][lane], get_timing_register_addr(lane,
286  i + 1,
287  slot,
288  rank),
289  9, 0);
290 
291  write_1d0(1, 0x103, 6, 1);
292  for (lane = 0; lane < 8; lane++)
293  write_500(info, channel,
294  info->training.
295  lane_timings[0][channel][slot][rank][lane],
296  get_timing_register_addr(lane, 0, slot, rank), 9, 0);
297 
298  for (i = 0; i < 2; i++) {
299  for (lane = 0; lane < 8; lane++)
300  write_500(info, channel,
301  info->training.lane_timings[i +
302  1][channel][slot]
303  [rank][lane], get_timing_register_addr(lane,
304  i + 1,
305  slot,
306  rank),
307  9, 0);
308  gav(get_580(channel, ((i + 1) << 2) | (rank << 5)));
309  }
310 
311  toggle_1d0_142_5ff();
312  write_1d0(0x2, 0x142, 3, 1);
313 
314  for (lane = 0; lane < 8; lane++) {
315  // printk (BIOS_ERR, "before: %x\n", info->training.lane_timings[2][channel][slot][rank][lane]);
316  info->training.lane_timings[2][channel][slot][rank][lane] =
317  read_500(info, channel,
318  get_timing_register_addr(lane, 2, slot, rank), 9);
319  //printk (BIOS_ERR, "after: %x\n", info->training.lane_timings[2][channel][slot][rank][lane]);
320  info->training.lane_timings[3][channel][slot][rank][lane] =
321  info->training.lane_timings[2][channel][slot][rank][lane] +
322  0x20;
323  }
324 }
325 
326 static int count_ranks_in_channel(struct raminfo *info, int channel)
327 {
328  int slot, rank;
329  int res = 0;
330  for (slot = 0; slot < NUM_SLOTS; slot++)
331  for (rank = 0; rank < NUM_SLOTS; rank++)
332  res += info->populated_ranks[channel][slot][rank];
333  return res;
334 }
335 
336 static void
337 config_rank(struct raminfo *info, int s3resume, int channel, int slot, int rank)
338 {
339  int add;
340 
341  write_1d0(0, 0x178, 7, 1);
342  seq9(info, channel, slot, rank);
343  program_timings(info, 0x80, channel, slot, rank);
344 
345  if (channel == 0)
346  add = count_ranks_in_channel(info, 1);
347  else
348  add = 0;
349  if (!s3resume)
350  gav(rw_test(rank + add));
351  program_timings(info, 0x00, channel, slot, rank);
352  if (!s3resume)
353  gav(rw_test(rank + add));
354  if (!s3resume)
355  gav(rw_test(rank + add));
356  write_1d0(0, 0x142, 3, 1);
357  write_1d0(0, 0x103, 6, 1);
358 
359  gav(get_580(channel, 0xc | (rank << 5)));
360  gav(read_1d0(0x142, 3));
361 
362  mchbar_write8(0x5ff, 0);
363  mchbar_write8(0x5ff, 1 << 7);
364 }
365 
366 static void set_4cf(struct raminfo *info, int channel, u8 bit, u8 val)
367 {
368  const u16 regtable[] = { 0x4cf, 0x659, 0x697 };
369 
370  val &= 1;
371  for (int i = 0; i < ARRAY_SIZE(regtable); i++)
372  rmw_500(info, channel, regtable[i], 4, ~(1 << bit), val << bit);
373 }
374 
375 static void set_334(int zero)
376 {
377  int j, k, channel;
378  const u32 val3[] = { 0x2a2b2a2b, 0x26272627, 0x2e2f2e2f, 0x2a2b };
379  u32 vd8[2][16];
380 
381  for (channel = 0; channel < NUM_CHANNELS; channel++) {
382  for (j = 0; j < 4; j++) {
383  u32 a = (j == 1) ? 0x29292929 : 0x31313131;
384  u32 lmask = (j == 3) ? 0xffff : 0xffffffff;
385  u16 c;
386  if ((j == 0 || j == 3) && zero)
387  c = 0;
388  else if (j == 3)
389  c = 0x5f;
390  else
391  c = 0x5f5f;
392 
393  for (k = 0; k < 2; k++) {
394  mchbar_write32(0x138 + 8 * k, channel << 26 | j << 24);
395  gav(vd8[1][(channel << 3) | (j << 1) | k] =
396  mchbar_read32(0x138 + 8 * k));
397  gav(vd8[0][(channel << 3) | (j << 1) | k] =
398  mchbar_read32(0x13c + 8 * k));
399  }
400 
401  mchbar_write32(0x334 + (channel << 10) + j * 0x44, zero ? 0 : val3[j]);
402  mchbar_write32(0x32c + (channel << 10) + j * 0x44,
403  zero ? 0 : 0x18191819 & lmask);
404  mchbar_write16(0x34a + (channel << 10) + j * 0x44, c);
405  mchbar_write32(0x33c + (channel << 10) + j * 0x44,
406  zero ? 0 : a & lmask);
407  mchbar_write32(0x344 + (channel << 10) + j * 0x44,
408  zero ? 0 : a & lmask);
409  }
410  }
411 
412  mchbar_setbits32(0x130, 1 << 0);
413  while (mchbar_read8(0x130) & 1)
414  ;
415 }
416 
417 static void rmw_1d0(u16 addr, u32 and, u32 or, int split)
418 {
419  u32 v;
420  v = read_1d0(addr, split);
421  write_1d0((v & and) | or, addr, split, 1);
422 }
423 
424 static int find_highest_bit_set(u16 val)
425 {
426  int i;
427  for (i = 15; i >= 0; i--)
428  if (val & (1 << i))
429  return i;
430  return -1;
431 }
432 
433 static int find_lowest_bit_set32(u32 val)
434 {
435  int i;
436  for (i = 0; i < 32; i++)
437  if (val & (1 << i))
438  return i;
439  return -1;
440 }
441 
442 enum {
443  DEVICE_TYPE = 2,
444  MODULE_TYPE = 3,
445  DENSITY = 4,
446  RANKS_AND_DQ = 7,
447  MEMORY_BUS_WIDTH = 8,
448  TIMEBASE_DIVIDEND = 10,
449  TIMEBASE_DIVISOR = 11,
450  CYCLETIME = 12,
451 
452  CAS_LATENCIES_LSB = 14,
453  CAS_LATENCIES_MSB = 15,
454  CAS_LATENCY_TIME = 16,
455  THERMAL_AND_REFRESH = 31,
456  REFERENCE_RAW_CARD_USED = 62,
457  RANK1_ADDRESS_MAPPING = 63
458 };
459 
460 static void calculate_timings(struct raminfo *info)
461 {
462  unsigned int cycletime;
463  unsigned int cas_latency_time;
464  unsigned int supported_cas_latencies;
465  unsigned int channel, slot;
466  unsigned int clock_speed_index;
467  unsigned int min_cas_latency;
468  unsigned int cas_latency;
469  unsigned int max_clock_index;
470 
471  /* Find common CAS latency */
472  supported_cas_latencies = 0x3fe;
473  for (channel = 0; channel < NUM_CHANNELS; channel++)
474  for (slot = 0; slot < NUM_SLOTS; slot++)
475  if (info->populated_ranks[channel][slot][0])
476  supported_cas_latencies &=
477  2 *
478  (info->
479  spd[channel][slot][CAS_LATENCIES_LSB] |
480  (info->
481  spd[channel][slot][CAS_LATENCIES_MSB] <<
482  8));
483 
484  max_clock_index = MIN(3, info->max_supported_clock_speed_index);
485 
486  cycletime = min_cycletime[max_clock_index];
487  cas_latency_time = min_cas_latency_time[max_clock_index];
488 
489  for (channel = 0; channel < NUM_CHANNELS; channel++)
490  for (slot = 0; slot < NUM_SLOTS; slot++)
491  if (info->populated_ranks[channel][slot][0]) {
492  unsigned int timebase;
493  timebase =
494  1000 *
495  info->
496  spd[channel][slot][TIMEBASE_DIVIDEND] /
497  info->spd[channel][slot][TIMEBASE_DIVISOR];
498  cycletime =
499  MAX(cycletime,
500  timebase *
501  info->spd[channel][slot][CYCLETIME]);
502  cas_latency_time =
503  MAX(cas_latency_time,
504  timebase *
505  info->
506  spd[channel][slot][CAS_LATENCY_TIME]);
507  }
508  if (cycletime > min_cycletime[0])
509  die("RAM init: Decoded SPD DRAM freq is slower than the controller minimum!");
510  for (clock_speed_index = 0; clock_speed_index < 3; clock_speed_index++) {
511  if (cycletime == min_cycletime[clock_speed_index])
512  break;
513  if (cycletime > min_cycletime[clock_speed_index]) {
514  clock_speed_index--;
515  cycletime = min_cycletime[clock_speed_index];
516  break;
517  }
518  }
519  min_cas_latency = DIV_ROUND_UP(cas_latency_time, cycletime);
520  cas_latency = 0;
521  while (supported_cas_latencies) {
522  cas_latency = find_highest_bit_set(supported_cas_latencies) + 3;
523  if (cas_latency <= min_cas_latency)
524  break;
525  supported_cas_latencies &=
526  ~(1 << find_highest_bit_set(supported_cas_latencies));
527  }
528 
529  if (cas_latency != min_cas_latency && clock_speed_index)
530  clock_speed_index--;
531 
532  if (cas_latency * min_cycletime[clock_speed_index] > 20000)
533  die("Couldn't configure DRAM");
534  info->clock_speed_index = clock_speed_index;
535  info->cas_latency = cas_latency;
536 }
537 
538 static void program_base_timings(struct raminfo *info)
539 {
540  unsigned int channel;
541  unsigned int slot, rank, lane;
542  unsigned int extended_silicon_revision;
543  int i;
544 
545  extended_silicon_revision = info->silicon_revision;
546  if (info->silicon_revision == 0)
547  for (channel = 0; channel < NUM_CHANNELS; channel++)
548  for (slot = 0; slot < NUM_SLOTS; slot++)
549  if ((info->
550  spd[channel][slot][MODULE_TYPE] & 0xF) ==
551  3)
552  extended_silicon_revision = 4;
553 
554  for (channel = 0; channel < NUM_CHANNELS; channel++) {
555  for (slot = 0; slot < NUM_SLOTS; slot++)
556  for (rank = 0; rank < NUM_SLOTS; rank++) {
557  int card_timing_2;
558  if (!info->populated_ranks[channel][slot][rank])
559  continue;
560 
561  for (lane = 0; lane < 9; lane++) {
562  int tm_reg;
563  int card_timing;
564 
565  card_timing = 0;
566  if ((info->
567  spd[channel][slot][MODULE_TYPE] &
568  0xF) == 3) {
569  int reference_card;
570  reference_card =
571  info->
572  spd[channel][slot]
573  [REFERENCE_RAW_CARD_USED] &
574  0x1f;
575  if (reference_card == 3)
576  card_timing =
577  u16_ffd1188[0][lane]
578  [info->
579  clock_speed_index];
580  if (reference_card == 5)
581  card_timing =
582  u16_ffd1188[1][lane]
583  [info->
584  clock_speed_index];
585  }
586 
587  info->training.
588  lane_timings[0][channel][slot][rank]
589  [lane] =
590  u8_FFFD1218[info->
591  clock_speed_index];
592  info->training.
593  lane_timings[1][channel][slot][rank]
594  [lane] = 256;
595 
596  for (tm_reg = 2; tm_reg < 4; tm_reg++)
597  info->training.
598  lane_timings[tm_reg]
599  [channel][slot][rank][lane]
600  =
601  u8_FFFD1240[channel]
602  [extended_silicon_revision]
603  [lane][2 * slot +
604  rank][info->
605  clock_speed_index]
606  + info->max4048[channel]
607  +
608  u8_FFFD0C78[channel]
609  [extended_silicon_revision]
610  [info->
611  mode4030[channel]][slot]
612  [rank][info->
613  clock_speed_index]
614  + card_timing;
615  for (tm_reg = 0; tm_reg < 4; tm_reg++)
616  write_500(info, channel,
617  info->training.
618  lane_timings[tm_reg]
619  [channel][slot][rank]
620  [lane],
621  get_timing_register_addr
622  (lane, tm_reg, slot,
623  rank), 9, 0);
624  }
625 
626  card_timing_2 = 0;
627  if (!(extended_silicon_revision != 4
628  || (info->
629  populated_ranks_mask[channel] & 5) ==
630  5)) {
631  if ((info->
632  spd[channel][slot]
633  [REFERENCE_RAW_CARD_USED] & 0x1F)
634  == 3)
635  card_timing_2 =
636  u16_FFFE0EB8[0][info->
637  clock_speed_index];
638  if ((info->
639  spd[channel][slot]
640  [REFERENCE_RAW_CARD_USED] & 0x1F)
641  == 5)
642  card_timing_2 =
643  u16_FFFE0EB8[1][info->
644  clock_speed_index];
645  }
646 
647  for (i = 0; i < 3; i++)
648  write_500(info, channel,
649  (card_timing_2 +
650  info->max4048[channel]
651  +
652  u8_FFFD0EF8[channel]
653  [extended_silicon_revision]
654  [info->
655  mode4030[channel]][info->
656  clock_speed_index]),
657  u16_fffd0c50[i][slot][rank],
658  8, 1);
659  write_500(info, channel,
660  (info->max4048[channel] +
661  u8_FFFD0C78[channel]
662  [extended_silicon_revision][info->
663  mode4030
664  [channel]]
665  [slot][rank][info->
666  clock_speed_index]),
667  u16_fffd0c70[slot][rank], 7, 1);
668  }
669  if (!info->populated_ranks_mask[channel])
670  continue;
671  for (i = 0; i < 3; i++)
672  write_500(info, channel,
673  (info->max4048[channel] +
674  info->avg4044[channel]
675  +
676  u8_FFFD17E0[channel]
677  [extended_silicon_revision][info->
678  mode4030
679  [channel]][info->
680  clock_speed_index]),
681  u16_fffd0c68[i], 8, 1);
682  }
683 }
684 
685 /* The time of clock cycle in ps. */
686 static unsigned int cycle_ps(struct raminfo *info)
687 {
688  return 2 * halfcycle_ps(info);
689 }
690 
691 /* Frequency in 0.1 MHz units. */
692 static unsigned int frequency_01(struct raminfo *info)
693 {
694  return 100 * frequency_11(info) / 9;
695 }
696 
697 static unsigned int ps_to_halfcycles(struct raminfo *info, unsigned int ps)
698 {
699  return (frequency_11(info) * 2) * ps / 900000;
700 }
701 
702 static unsigned int ns_to_cycles(struct raminfo *info, unsigned int ns)
703 {
704  return (frequency_11(info)) * ns / 900;
705 }
706 
707 static void compute_derived_timings(struct raminfo *info)
708 {
709  unsigned int channel, slot, rank;
710  int extended_silicon_revision;
711  int some_delay_1_ps;
712  int some_delay_2_ps;
713  int some_delay_2_halfcycles_ceil;
714  int some_delay_2_halfcycles_floor;
715  int some_delay_3_ps;
716  int some_delay_3_ps_rounded;
717  int some_delay_1_cycle_ceil;
718  int some_delay_1_cycle_floor;
719 
720  some_delay_3_ps_rounded = 0;
721  extended_silicon_revision = info->silicon_revision;
722  if (!info->silicon_revision)
723  for (channel = 0; channel < NUM_CHANNELS; channel++)
724  for (slot = 0; slot < NUM_SLOTS; slot++)
725  if ((info->
726  spd[channel][slot][MODULE_TYPE] & 0xF) ==
727  3)
728  extended_silicon_revision = 4;
729  if (info->board_lane_delay[7] < 5)
730  info->board_lane_delay[7] = 5;
731  info->revision_flag_1 = 2;
732  if (info->silicon_revision == 2 || info->silicon_revision == 3)
733  info->revision_flag_1 = 0;
734  if (info->revision < 16)
735  info->revision_flag_1 = 0;
736 
737  if (info->revision < 8)
738  info->revision_flag_1 = 0;
739  if (info->revision >= 8 && (info->silicon_revision == 0
740  || info->silicon_revision == 1))
741  some_delay_2_ps = 735;
742  else
743  some_delay_2_ps = 750;
744 
745  if (info->revision >= 0x10 && (info->silicon_revision == 0
746  || info->silicon_revision == 1))
747  some_delay_1_ps = 3929;
748  else
749  some_delay_1_ps = 3490;
750 
751  some_delay_1_cycle_floor = some_delay_1_ps / cycle_ps(info);
752  some_delay_1_cycle_ceil = some_delay_1_ps / cycle_ps(info);
753  if (some_delay_1_ps % cycle_ps(info))
754  some_delay_1_cycle_ceil++;
755  else
756  some_delay_1_cycle_floor--;
757  info->some_delay_1_cycle_floor = some_delay_1_cycle_floor;
758  if (info->revision_flag_1)
759  some_delay_2_ps = halfcycle_ps(info) >> 6;
760  some_delay_2_ps +=
761  MAX(some_delay_1_ps - 30,
762  2 * halfcycle_ps(info) * (some_delay_1_cycle_ceil - 1) + 1000) +
763  375;
764  some_delay_3_ps =
765  halfcycle_ps(info) - some_delay_2_ps % halfcycle_ps(info);
766  if (info->revision_flag_1) {
767  if (some_delay_3_ps >= 150) {
768  const int some_delay_3_halfcycles =
769  (some_delay_3_ps << 6) / halfcycle_ps(info);
770  some_delay_3_ps_rounded =
771  halfcycle_ps(info) * some_delay_3_halfcycles >> 6;
772  }
773  }
774  some_delay_2_halfcycles_ceil =
775  (some_delay_2_ps + halfcycle_ps(info) - 1) / halfcycle_ps(info) -
776  2 * (some_delay_1_cycle_ceil - 1);
777  if (info->revision_flag_1 && some_delay_3_ps < 150)
778  some_delay_2_halfcycles_ceil++;
779  some_delay_2_halfcycles_floor = some_delay_2_halfcycles_ceil;
780  if (info->revision < 0x10)
781  some_delay_2_halfcycles_floor =
782  some_delay_2_halfcycles_ceil - 1;
783  if (!info->revision_flag_1)
784  some_delay_2_halfcycles_floor++;
785  info->some_delay_2_halfcycles_ceil = some_delay_2_halfcycles_ceil;
786  info->some_delay_3_ps_rounded = some_delay_3_ps_rounded;
787  if ((info->populated_ranks[0][0][0] && info->populated_ranks[0][1][0])
788  || (info->populated_ranks[1][0][0]
789  && info->populated_ranks[1][1][0]))
790  info->max_slots_used_in_channel = 2;
791  else
792  info->max_slots_used_in_channel = 1;
793  for (channel = 0; channel < NUM_CHANNELS; channel++)
794  mchbar_write32(0x244 + (channel << 10),
795  ((info->revision < 8) ? 1 : 0x200) |
796  ((2 - info->max_slots_used_in_channel) << 17) |
797  (channel << 21) |
798  (info->some_delay_1_cycle_floor << 18) | 0x9510);
799  if (info->max_slots_used_in_channel == 1) {
800  info->mode4030[0] = (count_ranks_in_channel(info, 0) == 2);
801  info->mode4030[1] = (count_ranks_in_channel(info, 1) == 2);
802  } else {
803  info->mode4030[0] = ((count_ranks_in_channel(info, 0) == 1) || (count_ranks_in_channel(info, 0) == 2)) ? 2 : 3; /* 2 if 1 or 2 ranks */
804  info->mode4030[1] = ((count_ranks_in_channel(info, 1) == 1)
805  || (count_ranks_in_channel(info, 1) ==
806  2)) ? 2 : 3;
807  }
808  for (channel = 0; channel < NUM_CHANNELS; channel++) {
809  int max_of_unk;
810  int min_of_unk_2;
811 
812  int i, count;
813  int sum;
814 
815  if (!info->populated_ranks_mask[channel])
816  continue;
817 
818  max_of_unk = 0;
819  min_of_unk_2 = 32767;
820 
821  sum = 0;
822  count = 0;
823  for (i = 0; i < 3; i++) {
824  int unk1;
825  if (info->revision < 8)
826  unk1 =
827  u8_FFFD1891[0][channel][info->
828  clock_speed_index]
829  [i];
830  else if (!
831  (info->revision >= 0x10
832  || info->revision_flag_1))
833  unk1 =
834  u8_FFFD1891[1][channel][info->
835  clock_speed_index]
836  [i];
837  else
838  unk1 = 0;
839  for (slot = 0; slot < NUM_SLOTS; slot++)
840  for (rank = 0; rank < NUM_RANKS; rank++) {
841  int a = 0;
842  int b = 0;
843 
844  if (!info->
845  populated_ranks[channel][slot]
846  [rank])
847  continue;
848  if (extended_silicon_revision == 4
849  && (info->
850  populated_ranks_mask[channel] &
851  5) != 5) {
852  if ((info->
853  spd[channel][slot]
854  [REFERENCE_RAW_CARD_USED] &
855  0x1F) == 3) {
856  a = u16_ffd1178[0]
857  [info->
858  clock_speed_index];
859  b = u16_fe0eb8[0][info->
860  clock_speed_index];
861  } else
862  if ((info->
863  spd[channel][slot]
864  [REFERENCE_RAW_CARD_USED]
865  & 0x1F) == 5) {
866  a = u16_ffd1178[1]
867  [info->
868  clock_speed_index];
869  b = u16_fe0eb8[1][info->
870  clock_speed_index];
871  }
872  }
873  min_of_unk_2 = MIN(min_of_unk_2, a);
874  min_of_unk_2 = MIN(min_of_unk_2, b);
875  if (rank == 0) {
876  sum += a;
877  count++;
878  }
879  {
880  int t;
881  t = b +
882  u8_FFFD0EF8[channel]
883  [extended_silicon_revision]
884  [info->
885  mode4030[channel]][info->
886  clock_speed_index];
887  if (unk1 >= t)
888  max_of_unk =
889  MAX(max_of_unk,
890  unk1 - t);
891  }
892  }
893  {
894  int t =
895  u8_FFFD17E0[channel]
896  [extended_silicon_revision][info->
897  mode4030
898  [channel]]
899  [info->clock_speed_index] + min_of_unk_2;
900  if (unk1 >= t)
901  max_of_unk = MAX(max_of_unk, unk1 - t);
902  }
903  }
904 
905  if (count == 0)
906  die("No memory ranks found for channel %u\n", channel);
907 
908  info->avg4044[channel] = sum / count;
909  info->max4048[channel] = max_of_unk;
910  }
911 }
912 
913 static void jedec_read(struct raminfo *info,
914  int channel, int slot, int rank,
915  int total_rank, u8 addr3, unsigned int value)
916 {
917  /* Handle mirrored mapping. */
918  if ((rank & 1) && (info->spd[channel][slot][RANK1_ADDRESS_MAPPING] & 1)) {
919  addr3 = (addr3 & 0xCF) | ((addr3 & 0x10) << 1) | ((addr3 >> 1) & 0x10);
920  value = (value & ~0x1f8) | ((value >> 1) & 0xa8) | ((value & 0xa8) << 1);
921  }
922 
923  mchbar_clrsetbits8(0x271, 0x1f << 1, addr3);
924  mchbar_clrsetbits8(0x671, 0x1f << 1, addr3);
925 
926  read32p((value << 3) | (total_rank << 28));
927 
928  mchbar_clrsetbits8(0x271, 0x1f << 1, 1 << 1);
929  mchbar_clrsetbits8(0x671, 0x1f << 1, 1 << 1);
930 
931  read32p(total_rank << 28);
932 }
933 
934 enum {
935  MR1_RZQ12 = 512,
936  MR1_RZQ2 = 64,
937  MR1_RZQ4 = 4,
938  MR1_ODS34OHM = 2
939 };
940 
941 enum {
942  MR0_BT_INTERLEAVED = 8,
943  MR0_DLL_RESET_ON = 256
944 };
945 
946 enum {
947  MR2_RTT_WR_DISABLED = 0,
948  MR2_RZQ2 = 1 << 10
949 };
950 
951 static void jedec_init(struct raminfo *info)
952 {
953  int write_recovery;
954  int channel, slot, rank;
955  int total_rank;
956  int dll_on;
957  int self_refresh_temperature;
958  int auto_self_refresh;
959 
960  auto_self_refresh = 1;
961  self_refresh_temperature = 1;
962  if (info->board_lane_delay[3] <= 10) {
963  if (info->board_lane_delay[3] <= 8)
964  write_recovery = info->board_lane_delay[3] - 4;
965  else
966  write_recovery = 5;
967  } else {
968  write_recovery = 6;
969  }
970  FOR_POPULATED_RANKS {
971  auto_self_refresh &=
972  (info->spd[channel][slot][THERMAL_AND_REFRESH] >> 2) & 1;
973  self_refresh_temperature &=
974  info->spd[channel][slot][THERMAL_AND_REFRESH] & 1;
975  }
976  if (auto_self_refresh == 1)
977  self_refresh_temperature = 0;
978 
979  dll_on = ((info->silicon_revision != 2 && info->silicon_revision != 3)
980  || (info->populated_ranks[0][0][0]
981  && info->populated_ranks[0][1][0])
982  || (info->populated_ranks[1][0][0]
983  && info->populated_ranks[1][1][0]));
984 
985  total_rank = 0;
986 
987  for (channel = NUM_CHANNELS - 1; channel >= 0; channel--) {
988  int rtt, rtt_wr = MR2_RTT_WR_DISABLED;
989  int rzq_reg58e;
990 
991  if (info->silicon_revision == 2 || info->silicon_revision == 3) {
992  rzq_reg58e = 64;
993  rtt = MR1_RZQ2;
994  if (info->clock_speed_index != 0) {
995  rzq_reg58e = 4;
996  if (info->populated_ranks_mask[channel] == 3)
997  rtt = MR1_RZQ4;
998  }
999  } else {
1000  if ((info->populated_ranks_mask[channel] & 5) == 5) {
1001  rtt = MR1_RZQ12;
1002  rzq_reg58e = 64;
1003  rtt_wr = MR2_RZQ2;
1004  } else {
1005  rzq_reg58e = 4;
1006  rtt = MR1_RZQ4;
1007  }
1008  }
1009 
1010  mchbar_write16(0x588 + (channel << 10), 0);
1011  mchbar_write16(0x58a + (channel << 10), 4);
1012  mchbar_write16(0x58c + (channel << 10), rtt | MR1_ODS34OHM);
1013  mchbar_write16(0x58e + (channel << 10), rzq_reg58e | 0x82);
1014  mchbar_write16(0x590 + (channel << 10), 0x1282);
1015 
1016  for (slot = 0; slot < NUM_SLOTS; slot++)
1017  for (rank = 0; rank < NUM_RANKS; rank++)
1018  if (info->populated_ranks[channel][slot][rank]) {
1019  jedec_read(info, channel, slot, rank,
1020  total_rank, 0x28,
1021  rtt_wr | (info->
1022  clock_speed_index
1023  << 3)
1024  | (auto_self_refresh << 6) |
1025  (self_refresh_temperature <<
1026  7));
1027  jedec_read(info, channel, slot, rank,
1028  total_rank, 0x38, 0);
1029  jedec_read(info, channel, slot, rank,
1030  total_rank, 0x18,
1031  rtt | MR1_ODS34OHM);
1032  jedec_read(info, channel, slot, rank,
1033  total_rank, 6,
1034  (dll_on << 12) |
1035  (write_recovery << 9)
1036  | ((info->cas_latency - 4) <<
1037  4) | MR0_BT_INTERLEAVED |
1038  MR0_DLL_RESET_ON);
1039  total_rank++;
1040  }
1041  }
1042 }
1043 
1044 static void program_modules_memory_map(struct raminfo *info, int pre_jedec)
1045 {
1046  unsigned int channel, slot, rank;
1047  unsigned int total_mb[2] = { 0, 0 }; /* total memory per channel in MB */
1048  unsigned int channel_0_non_interleaved;
1049 
1050  FOR_ALL_RANKS {
1051  if (info->populated_ranks[channel][slot][rank]) {
1052  total_mb[channel] +=
1053  pre_jedec ? 256 : (256 << info->
1054  density[channel][slot] >> info->
1055  is_x16_module[channel][slot]);
1056  mchbar_write8(0x208 + rank + 2 * slot + (channel << 10),
1057  (pre_jedec ? (1 | ((1 + 1) << 1)) :
1058  (info->is_x16_module[channel][slot] |
1059  ((info->density[channel][slot] + 1) << 1))) |
1060  0x80);
1061  }
1062  mchbar_write16(0x200 + (channel << 10) + 4 * slot + 2 * rank,
1063  total_mb[channel] >> 6);
1064  }
1065 
1066  info->total_memory_mb = total_mb[0] + total_mb[1];
1067 
1068  info->interleaved_part_mb =
1069  pre_jedec ? 0 : 2 * MIN(total_mb[0], total_mb[1]);
1070  info->non_interleaved_part_mb =
1071  total_mb[0] + total_mb[1] - info->interleaved_part_mb;
1072  channel_0_non_interleaved = total_mb[0] - info->interleaved_part_mb / 2;
1073  mchbar_write32(0x100, channel_0_non_interleaved | info->non_interleaved_part_mb << 16);
1074  if (!pre_jedec)
1075  mchbar_write16(0x104, info->interleaved_part_mb);
1076 }
1077 
1078 static void program_board_delay(struct raminfo *info)
1079 {
1080  int cas_latency_shift;
1081  int some_delay_ns;
1082  int some_delay_3_half_cycles;
1083 
1084  unsigned int channel, i;
1085  int high_multiplier;
1086  int lane_3_delay;
1087  int cas_latency_derived;
1088 
1089  high_multiplier = 0;
1090  some_delay_ns = 200;
1091  some_delay_3_half_cycles = 4;
1092  cas_latency_shift = info->silicon_revision == 0
1093  || info->silicon_revision == 1 ? 1 : 0;
1094  if (info->revision < 8) {
1095  some_delay_ns = 600;
1096  cas_latency_shift = 0;
1097  }
1098  {
1099  int speed_bit;
1100  speed_bit =
1101  ((info->clock_speed_index > 1
1102  || (info->silicon_revision != 2
1103  && info->silicon_revision != 3))) ^ (info->revision >=
1104  0x10);
1105  write_500(info, 0, speed_bit | ((!info->use_ecc) << 1), 0x60e,
1106  3, 1);
1107  write_500(info, 1, speed_bit | ((!info->use_ecc) << 1), 0x60e,
1108  3, 1);
1109  if (info->revision >= 0x10 && info->clock_speed_index <= 1
1110  && (info->silicon_revision == 2
1111  || info->silicon_revision == 3))
1112  rmw_1d0(0x116, 5, 2, 4);
1113  }
1114  mchbar_write32(0x120, 1 << (info->max_slots_used_in_channel + 28) | 0x188e7f9f);
1115 
1116  mchbar_write8(0x124, info->board_lane_delay[4] + (frequency_01(info) + 999) / 1000);
1117  mchbar_write16(0x125, 0x1360);
1118  mchbar_write8(0x127, 0x40);
1119  if (info->fsb_frequency < frequency_11(info) / 2) {
1120  unsigned int some_delay_2_half_cycles;
1121  high_multiplier = 1;
1122  some_delay_2_half_cycles = ps_to_halfcycles(info,
1123  ((3 *
1124  fsbcycle_ps(info))
1125  >> 1) +
1126  (halfcycle_ps(info)
1127  *
1128  reg178_min[info->
1129  clock_speed_index]
1130  >> 6)
1131  +
1132  4 *
1133  halfcycle_ps(info)
1134  + 2230);
1135  some_delay_3_half_cycles =
1136  MIN((some_delay_2_half_cycles +
1137  (frequency_11(info) * 2) * (28 -
1138  some_delay_2_half_cycles) /
1139  (frequency_11(info) * 2 -
1140  4 * (info->fsb_frequency))) >> 3, 7);
1141  }
1142  if (mchbar_read8(0x2ca9) & 1)
1143  some_delay_3_half_cycles = 3;
1144  for (channel = 0; channel < NUM_CHANNELS; channel++) {
1145  mchbar_setbits32(0x220 + (channel << 10), 0x18001117);
1146  mchbar_write32(0x224 + (channel << 10),
1147  (info->max_slots_used_in_channel - 1) |
1148  (info->cas_latency - 5 - info->clock_speed_index)
1149  << 21 | (info->max_slots_used_in_channel +
1150  info->cas_latency - cas_latency_shift - 4) << 16 |
1151  (info->cas_latency - cas_latency_shift - 4) << 26 |
1152  (info->cas_latency - info->clock_speed_index +
1153  info->max_slots_used_in_channel - 6) << 8);
1154  mchbar_write32(0x228 + (channel << 10), info->max_slots_used_in_channel);
1155  mchbar_write8(0x239 + (channel << 10), 32);
1156  mchbar_write32(0x248 + (channel << 10), high_multiplier << 24 |
1157  some_delay_3_half_cycles << 25 | 0x840000);
1158  mchbar_write32(0x278 + (channel << 10), 0xc362042);
1159  mchbar_write32(0x27c + (channel << 10), 0x8b000062);
1160  mchbar_write32(0x24c + (channel << 10),
1161  (!!info->clock_speed_index) << 17 |
1162  ((2 + info->clock_speed_index -
1163  (!!info->clock_speed_index))) << 12 | 0x10200);
1164 
1165  mchbar_write8(0x267 + (channel << 10), 4);
1166  mchbar_write16(0x272 + (channel << 10), 0x155);
1167  mchbar_clrsetbits32(0x2bc + (channel << 10), 0xffffff, 0x707070);
1168 
1169  write_500(info, channel,
1170  ((!info->populated_ranks[channel][1][1])
1171  | (!info->populated_ranks[channel][1][0] << 1)
1172  | (!info->populated_ranks[channel][0][1] << 2)
1173  | (!info->populated_ranks[channel][0][0] << 3)),
1174  0x4c9, 4, 1);
1175  }
1176 
1177  mchbar_write8(0x2c4, (1 + (info->clock_speed_index != 0)) << 6 | 0xc);
1178  {
1179  u8 freq_divisor = 2;
1180  if (info->fsb_frequency == frequency_11(info))
1181  freq_divisor = 3;
1182  else if (2 * info->fsb_frequency < 3 * (frequency_11(info) / 2))
1183  freq_divisor = 1;
1184  else
1185  freq_divisor = 2;
1186  mchbar_write32(0x2c0, freq_divisor << 11 | 0x6009c400);
1187  }
1188 
1189  if (info->board_lane_delay[3] <= 10) {
1190  if (info->board_lane_delay[3] <= 8)
1191  lane_3_delay = info->board_lane_delay[3];
1192  else
1193  lane_3_delay = 10;
1194  } else {
1195  lane_3_delay = 12;
1196  }
1197  cas_latency_derived = info->cas_latency - info->clock_speed_index + 2;
1198  if (info->clock_speed_index > 1)
1199  cas_latency_derived++;
1200  for (channel = 0; channel < NUM_CHANNELS; channel++) {
1201  mchbar_write32(0x240 + (channel << 10),
1202  ((info->clock_speed_index == 0) * 0x11000) |
1203  0x1002100 | (2 + info->clock_speed_index) << 4 |
1204  (info->cas_latency - 3));
1205  write_500(info, channel, (info->clock_speed_index << 1) | 1,
1206  0x609, 6, 1);
1207  write_500(info, channel,
1208  info->clock_speed_index + 2 * info->cas_latency - 7,
1209  0x601, 6, 1);
1210 
1211  mchbar_write32(0x250 + (channel << 10),
1212  (lane_3_delay + info->clock_speed_index + 9) << 6 |
1213  info->board_lane_delay[7] << 2 |
1214  info->board_lane_delay[4] << 16 |
1215  info->board_lane_delay[1] << 25 |
1216  info->board_lane_delay[1] << 29 | 1);
1217  mchbar_write32(0x254 + (channel << 10),
1218  info->board_lane_delay[1] >> 3 |
1219  (info->board_lane_delay[8] + 4 * info->use_ecc) << 6 |
1220  0x80 | info->board_lane_delay[6] << 1 |
1221  info->board_lane_delay[2] << 28 |
1222  cas_latency_derived << 16 | 0x4700000);
1223  mchbar_write32(0x258 + (channel << 10),
1224  (info->board_lane_delay[5] + info->clock_speed_index + 9) << 12 |
1225  (info->clock_speed_index - info->cas_latency + 12) << 8 |
1226  info->board_lane_delay[2] << 17 |
1227  info->board_lane_delay[4] << 24 | 0x47);
1228  mchbar_write32(0x25c + (channel << 10),
1229  info->board_lane_delay[1] << 1 |
1230  info->board_lane_delay[0] << 8 | 0x1da50000);
1231  mchbar_write8(0x264 + (channel << 10), 0xff);
1232  mchbar_write8(0x5f8 + (channel << 10), cas_latency_shift << 3 | info->use_ecc);
1233  }
1234 
1235  program_modules_memory_map(info, 1);
1236 
1237  mchbar_clrsetbits16(0x610, 0xfe3c,
1238  MIN(ns_to_cycles(info, some_delay_ns) / 2, 127) << 9 | 0x3c);
1239  mchbar_setbits16(0x612, 1 << 8);
1240  mchbar_setbits16(0x214, 0x3e00);
1241  for (i = 0; i < 8; i++) {
1242  pci_write_config32(QPI_SAD, SAD_DRAM_RULE(i),
1243  (info->total_memory_mb - 64) | !i | 2);
1244  pci_write_config32(QPI_SAD, SAD_INTERLEAVE_LIST(i), 0);
1245  }
1246 }
1247 
1248 #define DEFAULT_PCI_MMIO_SIZE 2048
1249 
1250 static void program_total_memory_map(struct raminfo *info)
1251 {
1252  unsigned int tom, tolud, touud;
1253  unsigned int quickpath_reserved;
1254  unsigned int remap_base;
1255  unsigned int uma_base_igd;
1256  unsigned int uma_base_gtt;
1257  unsigned int mmio_size;
1258  int memory_remap;
1259  unsigned int memory_map[8];
1260  int i;
1261  unsigned int current_limit;
1262  unsigned int tseg_base;
1263  int uma_size_igd = 0, uma_size_gtt = 0;
1264 
1265  memset(memory_map, 0, sizeof(memory_map));
1266 
1267  if (info->uma_enabled) {
1268  u16 t = pci_read_config16(NORTHBRIDGE, GGC);
1269  gav(t);
1270  const int uma_sizes_gtt[16] =
1271  { 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 3, 4, 42, 42, 42, 42 };
1272  /* Igd memory */
1273  const int uma_sizes_igd[16] = {
1274  0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352,
1275  256, 512
1276  };
1277 
1278  uma_size_igd = uma_sizes_igd[(t >> 4) & 0xF];
1279  uma_size_gtt = uma_sizes_gtt[(t >> 8) & 0xF];
1280  }
1281 
1282  mmio_size = DEFAULT_PCI_MMIO_SIZE;
1283 
1284  tom = info->total_memory_mb;
1285  if (tom == 4096)
1286  tom = 4032;
1287  touud = ALIGN_DOWN(tom - info->memory_reserved_for_heci_mb, 64);
1288  tolud = ALIGN_DOWN(MIN(4096 - mmio_size + ALIGN_UP(uma_size_igd + uma_size_gtt, 64)
1289  , touud), 64);
1290  memory_remap = 0;
1291  if (touud - tolud > 64) {
1292  memory_remap = 1;
1293  remap_base = MAX(4096, touud);
1294  touud = touud - tolud + 4096;
1295  }
1296  if (touud > 4096)
1297  memory_map[2] = touud | 1;
1298  quickpath_reserved = 0;
1299 
1300  u32 t = pci_read_config32(QPI_SAD, 0x68);
1301 
1302  gav(t);
1303 
1304  if (t & 0x800) {
1305  u32 shift = t >> 20;
1306  if (shift == 0)
1307  die("Quickpath value is 0\n");
1308  quickpath_reserved = (u32)1 << find_lowest_bit_set32(shift);
1309  }
1310 
1311  if (memory_remap)
1312  touud -= quickpath_reserved;
1313 
1314  uma_base_igd = tolud - uma_size_igd;
1315  uma_base_gtt = uma_base_igd - uma_size_gtt;
1316  tseg_base = ALIGN_DOWN(uma_base_gtt, 64) - (CONFIG_SMM_TSEG_SIZE >> 20);
1317  if (!memory_remap)
1318  tseg_base -= quickpath_reserved;
1319  tseg_base = ALIGN_DOWN(tseg_base, 8);
1320 
1321  pci_write_config16(NORTHBRIDGE, TOLUD, tolud << 4);
1322  pci_write_config16(NORTHBRIDGE, TOM, tom >> 6);
1323  if (memory_remap) {
1324  pci_write_config16(NORTHBRIDGE, REMAPBASE, remap_base >> 6);
1325  pci_write_config16(NORTHBRIDGE, REMAPLIMIT, (touud - 64) >> 6);
1326  }
1327  pci_write_config16(NORTHBRIDGE, TOUUD, touud);
1328 
1329  if (info->uma_enabled) {
1330  pci_write_config32(NORTHBRIDGE, IGD_BASE, uma_base_igd << 20);
1331  pci_write_config32(NORTHBRIDGE, GTT_BASE, uma_base_gtt << 20);
1332  }
1333  pci_write_config32(NORTHBRIDGE, TSEG, tseg_base << 20);
1334 
1335  current_limit = 0;
1336  memory_map[0] = ALIGN_DOWN(uma_base_gtt, 64) | 1;
1337  memory_map[1] = 4096;
1338  for (i = 0; i < ARRAY_SIZE(memory_map); i++) {
1339  current_limit = MAX(current_limit, memory_map[i] & ~1);
1340  pci_write_config32(QPI_SAD, SAD_DRAM_RULE(i),
1341  (memory_map[i] & 1) | ALIGN_DOWN(current_limit -
1342  1, 64) | 2);
1343  pci_write_config32(QPI_SAD, SAD_INTERLEAVE_LIST(i), 0);
1344  }
1345 }
1346 
1347 static void collect_system_info(struct raminfo *info)
1348 {
1349  u32 capid0[3];
1350  int i;
1351  unsigned int channel;
1352 
1353  for (i = 0; i < 3; i++) {
1354  capid0[i] = pci_read_config32(NORTHBRIDGE, CAPID0 | (i << 2));
1355  printk(BIOS_DEBUG, "CAPID0[%d] = 0x%08x\n", i, capid0[i]);
1356  }
1357  info->revision = pci_read_config8(NORTHBRIDGE, PCI_REVISION_ID);
1358  printk(BIOS_DEBUG, "Revision ID: 0x%x\n", info->revision);
1359  printk(BIOS_DEBUG, "Device ID: 0x%x\n", pci_read_config16(NORTHBRIDGE, PCI_DEVICE_ID));
1360 
1361  info->max_supported_clock_speed_index = (~capid0[1] & 7);
1362 
1363  if ((capid0[1] >> 11) & 1)
1364  info->uma_enabled = 0;
1365  else
1366  gav(info->uma_enabled =
1367  pci_read_config8(NORTHBRIDGE, DEVEN) & 8);
1368  /* Unrecognised: [0000:fffd3d2d] 37f81.37f82 ! CPUID: eax: 00000001; ecx: 00000e00 => 00020655.00010800.029ae3ff.bfebfbff */
1369  info->silicon_revision = 0;
1370 
1371  if (capid0[2] & 2) {
1372  info->silicon_revision = 0;
1373  info->max_supported_clock_speed_index = 2;
1374  for (channel = 0; channel < NUM_CHANNELS; channel++)
1375  if (info->populated_ranks[channel][0][0]
1376  && (info->spd[channel][0][MODULE_TYPE] & 0xf) ==
1377  3) {
1378  info->silicon_revision = 2;
1379  info->max_supported_clock_speed_index = 1;
1380  }
1381  } else {
1382  switch (((capid0[2] >> 18) & 1) + 2 * ((capid0[1] >> 3) & 1)) {
1383  case 1:
1384  case 2:
1385  info->silicon_revision = 3;
1386  break;
1387  case 3:
1388  info->silicon_revision = 0;
1389  break;
1390  case 0:
1391  info->silicon_revision = 2;
1392  break;
1393  }
1394  switch (pci_read_config16(NORTHBRIDGE, PCI_DEVICE_ID)) {
1395  case 0x40:
1396  info->silicon_revision = 0;
1397  break;
1398  case 0x48:
1399  info->silicon_revision = 1;
1400  break;
1401  }
1402  }
1403 }
1404 
1405 static void write_training_data(struct raminfo *info)
1406 {
1407  int tm, channel, slot, rank, lane;
1408  if (info->revision < 8)
1409  return;
1410 
1411  for (tm = 0; tm < 4; tm++)
1412  for (channel = 0; channel < NUM_CHANNELS; channel++)
1413  for (slot = 0; slot < NUM_SLOTS; slot++)
1414  for (rank = 0; rank < NUM_RANKS; rank++)
1415  for (lane = 0; lane < 9; lane++)
1416  write_500(info, channel,
1417  info->
1418  cached_training->
1419  lane_timings[tm]
1420  [channel][slot][rank]
1421  [lane],
1422  get_timing_register_addr
1423  (lane, tm, slot,
1424  rank), 9, 0);
1425  write_1d0(info->cached_training->reg_178, 0x178, 7, 1);
1426  write_1d0(info->cached_training->reg_10b, 0x10b, 6, 1);
1427 }
1428 
1429 static void dump_timings(struct raminfo *info)
1430 {
1431  int channel, slot, rank, lane, i;
1432  printk(RAM_SPEW, "Timings:\n");
1433  FOR_POPULATED_RANKS {
1434  printk(RAM_SPEW, "channel %d, slot %d, rank %d\n", channel,
1435  slot, rank);
1436  for (lane = 0; lane < 9; lane++) {
1437  printk(RAM_SPEW, "lane %d: ", lane);
1438  for (i = 0; i < 4; i++) {
1439  printk(RAM_SPEW, "%x (%x) ",
1440  read_500(info, channel,
1441  get_timing_register_addr
1442  (lane, i, slot, rank),
1443  9),
1444  info->training.
1445  lane_timings[i][channel][slot][rank]
1446  [lane]);
1447  }
1448  printk(RAM_SPEW, "\n");
1449  }
1450  }
1451  printk(RAM_SPEW, "[178] = %x (%x)\n", read_1d0(0x178, 7),
1452  info->training.reg_178);
1453  printk(RAM_SPEW, "[10b] = %x (%x)\n", read_1d0(0x10b, 6),
1454  info->training.reg_10b);
1455 }
1456 
1457 /* Read timings and other registers that need to be restored verbatim and
1458  put them to CBMEM.
1459  */
1460 static void save_timings(struct raminfo *info)
1461 {
1462  struct ram_training train;
1463  int channel, slot, rank, lane, i;
1464 
1465  train = info->training;
1466  FOR_POPULATED_RANKS for (lane = 0; lane < 9; lane++)
1467  for (i = 0; i < 4; i++)
1468  train.lane_timings[i][channel][slot][rank][lane] =
1469  read_500(info, channel,
1470  get_timing_register_addr(lane, i, slot,
1471  rank), 9);
1472  train.reg_178 = read_1d0(0x178, 7);
1473  train.reg_10b = read_1d0(0x10b, 6);
1474 
1475  for (channel = 0; channel < NUM_CHANNELS; channel++) {
1476  u32 reg32;
1477  reg32 = mchbar_read32((channel << 10) + 0x274);
1478  train.reg274265[channel][0] = reg32 >> 16;
1479  train.reg274265[channel][1] = reg32 & 0xffff;
1480  train.reg274265[channel][2] = mchbar_read16((channel << 10) + 0x265) >> 8;
1481  }
1482  train.reg2ca9_bit0 = mchbar_read8(0x2ca9) & 1;
1483  train.reg_6dc = mchbar_read32(0x6dc);
1484  train.reg_6e8 = mchbar_read32(0x6e8);
1485 
1486  printk(RAM_SPEW, "[6dc] = %x\n", train.reg_6dc);
1487  printk(RAM_SPEW, "[6e8] = %x\n", train.reg_6e8);
1488 
1489  /* Save the MRC S3 restore data to cbmem */
1490  mrc_cache_stash_data(MRC_TRAINING_DATA, MRC_CACHE_VERSION,
1491  &train, sizeof(train));
1492 }
1493 
1494 static const struct ram_training *get_cached_training(void)
1495 {
1496  return mrc_cache_current_mmap_leak(MRC_TRAINING_DATA,
1497  MRC_CACHE_VERSION,
1498  NULL);
1499 }
1500 
1501 static int have_match_ranks(struct raminfo *info, int channel, int ranks)
1502 {
1503  int ranks_in_channel;
1504  ranks_in_channel = info->populated_ranks[channel][0][0]
1505  + info->populated_ranks[channel][0][1]
1506  + info->populated_ranks[channel][1][0]
1507  + info->populated_ranks[channel][1][1];
1508 
1509  /* empty channel */
1510  if (ranks_in_channel == 0)
1511  return 1;
1512 
1513  if (ranks_in_channel != ranks)
1514  return 0;
1515  /* single slot */
1516  if (info->populated_ranks[channel][0][0] !=
1517  info->populated_ranks[channel][1][0])
1518  return 1;
1519  if (info->populated_ranks[channel][0][1] !=
1520  info->populated_ranks[channel][1][1])
1521  return 1;
1522  if (info->is_x16_module[channel][0] != info->is_x16_module[channel][1])
1523  return 0;
1524  if (info->density[channel][0] != info->density[channel][1])
1525  return 0;
1526  return 1;
1527 }
1528 
1529 static void read_4090(struct raminfo *info)
1530 {
1531  int i, channel, slot, rank, lane;
1532  for (i = 0; i < 2; i++)
1533  for (slot = 0; slot < NUM_SLOTS; slot++)
1534  for (rank = 0; rank < NUM_RANKS; rank++)
1535  for (lane = 0; lane < 9; lane++)
1536  info->training.
1537  lane_timings[0][i][slot][rank][lane]
1538  = 32;
1539 
1540  for (i = 1; i < 4; i++)
1541  for (channel = 0; channel < NUM_CHANNELS; channel++)
1542  for (slot = 0; slot < NUM_SLOTS; slot++)
1543  for (rank = 0; rank < NUM_RANKS; rank++)
1544  for (lane = 0; lane < 9; lane++) {
1545  info->training.
1546  lane_timings[i][channel]
1547  [slot][rank][lane] =
1548  read_500(info, channel,
1549  get_timing_register_addr
1550  (lane, i, slot,
1551  rank), 9)
1552  + (i == 1) * 11; // !!!!
1553  }
1554 
1555 }
1556 
1557 static u32 get_etalon2(int flip, u32 addr)
1558 {
1559  const u16 invmask[] = {
1560  0xaaaa, 0x6db6, 0x4924, 0xeeee, 0xcccc, 0x8888, 0x7bde, 0x739c,
1561  0x6318, 0x4210, 0xefbe, 0xcf3c, 0x8e38, 0x0c30, 0x0820
1562  };
1563  u32 ret;
1564  u32 comp4 = addr / 480;
1565  addr %= 480;
1566  u32 comp1 = addr & 0xf;
1567  u32 comp2 = (addr >> 4) & 1;
1568  u32 comp3 = addr >> 5;
1569 
1570  if (comp4)
1571  ret = 0x1010101 << (comp4 - 1);
1572  else
1573  ret = 0;
1574  if (flip ^ (((invmask[comp3] >> comp1) ^ comp2) & 1))
1575  ret = ~ret;
1576 
1577  return ret;
1578 }
1579 
1580 static void disable_cache_region(void)
1581 {
1582  msr_t msr = {.lo = 0, .hi = 0 };
1583 
1584  wrmsr(MTRR_PHYS_BASE(3), msr);
1585  wrmsr(MTRR_PHYS_MASK(3), msr);
1586 }
1587 
1588 static void enable_cache_region(unsigned int base, unsigned int size)
1589 {
1590  msr_t msr;
1591  msr.lo = base | MTRR_TYPE_WRPROT;
1592  msr.hi = 0;
1593  wrmsr(MTRR_PHYS_BASE(3), msr);
1594  msr.lo = ((~(ALIGN_DOWN(size + 4096, 4096) - 1) | MTRR_DEF_TYPE_EN)
1595  & 0xffffffff);
1596  msr.hi = 0x0000000f;
1597  wrmsr(MTRR_PHYS_MASK(3), msr);
1598 }
1599 
1600 static void flush_cache(u32 start, u32 size)
1601 {
1602  u32 end;
1603  u32 addr;
1604 
1605  end = start + (ALIGN_DOWN(size + 4096, 4096));
1606  for (addr = start; addr < end; addr += 64)
1607  clflush((void *)(uintptr_t)addr);
1608 }
1609 
1610 static void clear_errors(void)
1611 {
1612  pci_write_config8(NORTHBRIDGE, 0xc0, 0x01);
1613 }
1614 
1615 static void write_testing(struct raminfo *info, int totalrank, int flip)
1616 {
1617  int nwrites = 0;
1618  /* in 8-byte units. */
1619  u32 offset;
1620  u8 *base;
1621 
1622  base = (u8 *)(uintptr_t)(totalrank << 28);
1623  for (offset = 0; offset < 9 * 480; offset += 2) {
1624  write32(base + offset * 8, get_etalon2(flip, offset));
1625  write32(base + offset * 8 + 4, get_etalon2(flip, offset));
1626  write32(base + offset * 8 + 8, get_etalon2(flip, offset + 1));
1627  write32(base + offset * 8 + 12, get_etalon2(flip, offset + 1));
1628  nwrites += 4;
1629  if (nwrites >= 320) {
1630  clear_errors();
1631  nwrites = 0;
1632  }
1633  }
1634 }
1635 
1636 static u8 check_testing(struct raminfo *info, u8 total_rank, int flip)
1637 {
1638  u8 failmask = 0;
1639  int i;
1640  int comp1, comp2, comp3;
1641  u32 failxor[2] = { 0, 0 };
1642 
1643  enable_cache_region((total_rank << 28), 1728 * 5 * 4);
1644 
1645  for (comp3 = 0; comp3 < 9 && failmask != 0xff; comp3++) {
1646  for (comp1 = 0; comp1 < 4; comp1++)
1647  for (comp2 = 0; comp2 < 60; comp2++) {
1648  u32 re[4];
1649  u32 curroffset =
1650  comp3 * 8 * 60 + 2 * comp1 + 8 * comp2;
1651  read128((total_rank << 28) | (curroffset << 3),
1652  (u64 *) re);
1653  failxor[0] |=
1654  get_etalon2(flip, curroffset) ^ re[0];
1655  failxor[1] |=
1656  get_etalon2(flip, curroffset) ^ re[1];
1657  failxor[0] |=
1658  get_etalon2(flip, curroffset | 1) ^ re[2];
1659  failxor[1] |=
1660  get_etalon2(flip, curroffset | 1) ^ re[3];
1661  }
1662  for (i = 0; i < 8; i++)
1663  if ((0xff << (8 * (i % 4))) & failxor[i / 4])
1664  failmask |= 1 << i;
1665  }
1666  disable_cache_region();
1667  flush_cache((total_rank << 28), 1728 * 5 * 4);
1668  return failmask;
1669 }
1670 
1671 const u32 seed1[0x18] = {
1672  0x3a9d5ab5, 0x576cb65b, 0x555773b6, 0x2ab772ee,
1673  0x555556ee, 0x3a9d5ab5, 0x576cb65b, 0x555773b6,
1674  0x2ab772ee, 0x555556ee, 0x5155a555, 0x5155a555,
1675  0x5155a555, 0x5155a555, 0x3a9d5ab5, 0x576cb65b,
1676  0x555773b6, 0x2ab772ee, 0x555556ee, 0x55d6b4a5,
1677  0x366d6b3a, 0x2ae5ddbb, 0x3b9ddbb7, 0x55d6b4a5,
1678 };
1679 
1680 static u32 get_seed2(int a, int b)
1681 {
1682  const u32 seed2[5] = {
1683  0x55555555, 0x33333333, 0x2e555a55, 0x55555555,
1684  0x5b6db6db,
1685  };
1686  u32 r;
1687  r = seed2[(a + (a >= 10)) / 5];
1688  return b ? ~r : r;
1689 }
1690 
1691 static int make_shift(int comp2, int comp5, int x)
1692 {
1693  const u8 seed3[32] = {
1694  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1695  0x00, 0x00, 0x38, 0x1c, 0x3c, 0x18, 0x38, 0x38,
1696  0x38, 0x38, 0x38, 0x38, 0x0f, 0x0f, 0x0f, 0x0f,
1697  0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
1698  };
1699 
1700  return (comp2 - ((seed3[comp5] >> (x & 7)) & 1)) & 0x1f;
1701 }
1702 
1703 static u32 get_etalon(int flip, u32 addr)
1704 {
1705  u32 mask_byte = 0;
1706  int comp1 = (addr >> 1) & 1;
1707  int comp2 = (addr >> 3) & 0x1f;
1708  int comp3 = (addr >> 8) & 0xf;
1709  int comp4 = (addr >> 12) & 0xf;
1710  int comp5 = (addr >> 16) & 0x1f;
1711  u32 mask_bit = ~(0x10001 << comp3);
1712  u32 part1;
1713  u32 part2;
1714  int byte;
1715 
1716  part2 =
1717  ((seed1[comp5] >>
1718  make_shift(comp2, comp5,
1719  (comp3 >> 3) | (comp1 << 2) | 2)) & 1) ^ flip;
1720  part1 =
1721  ((seed1[comp5] >>
1722  make_shift(comp2, comp5,
1723  (comp3 >> 3) | (comp1 << 2) | 0)) & 1) ^ flip;
1724 
1725  for (byte = 0; byte < 4; byte++)
1726  if ((get_seed2(comp5, comp4) >>
1727  make_shift(comp2, comp5, (byte | (comp1 << 2)))) & 1)
1728  mask_byte |= 0xff << (8 * byte);
1729 
1730  return (mask_bit & mask_byte) | (part1 << comp3) | (part2 <<
1731  (comp3 + 16));
1732 }
1733 
1734 static void
1735 write_testing_type2(struct raminfo *info, u8 totalrank, u8 region, u8 block,
1736  char flip)
1737 {
1738  int i;
1739  for (i = 0; i < 2048; i++)
1740  write32p((totalrank << 28) | (region << 25) | (block << 16) |
1741  (i << 2), get_etalon(flip, (block << 16) | (i << 2)));
1742 }
1743 
1744 static u8
1745 check_testing_type2(struct raminfo *info, u8 totalrank, u8 region, u8 block,
1746  char flip)
1747 {
1748  u8 failmask = 0;
1749  u32 failxor[2];
1750  int i;
1751  int comp1, comp2, comp3;
1752 
1753  failxor[0] = 0;
1754  failxor[1] = 0;
1755 
1756  enable_cache_region(totalrank << 28, 134217728);
1757  for (comp3 = 0; comp3 < 2 && failmask != 0xff; comp3++) {
1758  for (comp1 = 0; comp1 < 16; comp1++)
1759  for (comp2 = 0; comp2 < 64; comp2++) {
1760  u32 addr =
1761  (totalrank << 28) | (region << 25) | (block
1762  << 16)
1763  | (comp3 << 12) | (comp2 << 6) | (comp1 <<
1764  2);
1765  failxor[comp1 & 1] |=
1766  read32p(addr) ^ get_etalon(flip, addr);
1767  }
1768  for (i = 0; i < 8; i++)
1769  if ((0xff << (8 * (i % 4))) & failxor[i / 4])
1770  failmask |= 1 << i;
1771  }
1772  disable_cache_region();
1773  flush_cache((totalrank << 28) | (region << 25) | (block << 16), 16384);
1774  return failmask;
1775 }
1776 
1777 static int check_bounded(unsigned short *vals, u16 bound)
1778 {
1779  int i;
1780 
1781  for (i = 0; i < 8; i++)
1782  if (vals[i] < bound)
1783  return 0;
1784  return 1;
1785 }
1786 
1787 enum state {
1788  BEFORE_USABLE = 0, AT_USABLE = 1, AT_MARGIN = 2, COMPLETE = 3
1789 };
1790 
1791 static int validate_state(enum state *in)
1792 {
1793  int i;
1794  for (i = 0; i < 8; i++)
1795  if (in[i] != COMPLETE)
1796  return 0;
1797  return 1;
1798 }
1799 
1800 static void
1801 do_fsm(enum state *state, u16 *counter,
1802  u8 fail_mask, int margin, int uplimit,
1803  u8 *res_low, u8 *res_high, u8 val)
1804 {
1805  int lane;
1806 
1807  for (lane = 0; lane < 8; lane++) {
1808  int is_fail = (fail_mask >> lane) & 1;
1809  switch (state[lane]) {
1810  case BEFORE_USABLE:
1811  if (!is_fail) {
1812  counter[lane] = 1;
1813  state[lane] = AT_USABLE;
1814  break;
1815  }
1816  counter[lane] = 0;
1817  state[lane] = BEFORE_USABLE;
1818  break;
1819  case AT_USABLE:
1820  if (!is_fail) {
1821  ++counter[lane];
1822  if (counter[lane] >= margin) {
1823  state[lane] = AT_MARGIN;
1824  res_low[lane] = val - margin + 1;
1825  break;
1826  }
1827  state[lane] = 1;
1828  break;
1829  }
1830  counter[lane] = 0;
1831  state[lane] = BEFORE_USABLE;
1832  break;
1833  case AT_MARGIN:
1834  if (is_fail) {
1835  state[lane] = COMPLETE;
1836  res_high[lane] = val - 1;
1837  } else {
1838  counter[lane]++;
1839  state[lane] = AT_MARGIN;
1840  if (val == uplimit) {
1841  state[lane] = COMPLETE;
1842  res_high[lane] = uplimit;
1843  }
1844  }
1845  break;
1846  case COMPLETE:
1847  break;
1848  }
1849  }
1850 }
1851 
1852 static void
1853 train_ram_at_178(struct raminfo *info, u8 channel, int slot, int rank,
1854  u8 total_rank, u8 reg_178, int first_run, int niter,
1855  timing_bounds_t * timings)
1856 {
1857  int lane;
1858  enum state state[8];
1859  u16 count[8];
1860  u8 lower_usable[8];
1861  u8 upper_usable[8];
1862  unsigned short num_successfully_checked[8];
1863  u8 reg1b3;
1864  int i;
1865 
1866  for (i = 0; i < 8; i++)
1867  state[i] = BEFORE_USABLE;
1868 
1869  if (!first_run) {
1870  int is_all_ok = 1;
1871  for (lane = 0; lane < 8; lane++)
1872  if (timings[reg_178][channel][slot][rank][lane].
1873  smallest ==
1874  timings[reg_178][channel][slot][rank][lane].
1875  largest) {
1876  timings[reg_178][channel][slot][rank][lane].
1877  smallest = 0;
1878  timings[reg_178][channel][slot][rank][lane].
1879  largest = 0;
1880  is_all_ok = 0;
1881  }
1882  if (is_all_ok) {
1883  for (i = 0; i < 8; i++)
1884  state[i] = COMPLETE;
1885  }
1886  }
1887 
1888  for (reg1b3 = 0; reg1b3 < 0x30 && !validate_state(state); reg1b3++) {
1889  u8 failmask = 0;
1890  write_1d0(reg1b3 ^ 32, 0x1b3, 6, 1);
1891  write_1d0(reg1b3 ^ 32, 0x1a3, 6, 1);
1892  failmask = check_testing(info, total_rank, 0);
1893  mchbar_setbits32(0xfb0, 3 << 16);
1894  do_fsm(state, count, failmask, 5, 47, lower_usable,
1895  upper_usable, reg1b3);
1896  }
1897 
1898  if (reg1b3) {
1899  write_1d0(0, 0x1b3, 6, 1);
1900  write_1d0(0, 0x1a3, 6, 1);
1901  for (lane = 0; lane < 8; lane++) {
1902  if (state[lane] == COMPLETE) {
1903  timings[reg_178][channel][slot][rank][lane].
1904  smallest =
1905  lower_usable[lane] +
1906  (info->training.
1907  lane_timings[0][channel][slot][rank][lane]
1908  & 0x3F) - 32;
1909  timings[reg_178][channel][slot][rank][lane].
1910  largest =
1911  upper_usable[lane] +
1912  (info->training.
1913  lane_timings[0][channel][slot][rank][lane]
1914  & 0x3F) - 32;
1915  }
1916  }
1917  }
1918 
1919  if (!first_run) {
1920  for (lane = 0; lane < 8; lane++)
1921  if (state[lane] == COMPLETE) {
1922  write_500(info, channel,
1923  timings[reg_178][channel][slot][rank]
1924  [lane].smallest,
1925  get_timing_register_addr(lane, 0,
1926  slot, rank),
1927  9, 1);
1928  write_500(info, channel,
1929  timings[reg_178][channel][slot][rank]
1930  [lane].smallest +
1931  info->training.
1932  lane_timings[1][channel][slot][rank]
1933  [lane]
1934  -
1935  info->training.
1936  lane_timings[0][channel][slot][rank]
1937  [lane], get_timing_register_addr(lane,
1938  1,
1939  slot,
1940  rank),
1941  9, 1);
1942  num_successfully_checked[lane] = 0;
1943  } else
1944  num_successfully_checked[lane] = -1;
1945 
1946  do {
1947  u8 failmask = 0;
1948  for (i = 0; i < niter; i++) {
1949  if (failmask == 0xFF)
1950  break;
1951  failmask |=
1952  check_testing_type2(info, total_rank, 2, i,
1953  0);
1954  failmask |=
1955  check_testing_type2(info, total_rank, 3, i,
1956  1);
1957  }
1958  mchbar_setbits32(0xfb0, 3 << 16);
1959  for (lane = 0; lane < 8; lane++)
1960  if (num_successfully_checked[lane] != 0xffff) {
1961  if ((1 << lane) & failmask) {
1962  if (timings[reg_178][channel]
1963  [slot][rank][lane].
1964  largest <=
1965  timings[reg_178][channel]
1966  [slot][rank][lane].smallest)
1967  num_successfully_checked
1968  [lane] = -1;
1969  else {
1970  num_successfully_checked
1971  [lane] = 0;
1972  timings[reg_178]
1973  [channel][slot]
1974  [rank][lane].
1975  smallest++;
1976  write_500(info, channel,
1977  timings
1978  [reg_178]
1979  [channel]
1980  [slot][rank]
1981  [lane].
1982  smallest,
1983  get_timing_register_addr
1984  (lane, 0,
1985  slot, rank),
1986  9, 1);
1987  write_500(info, channel,
1988  timings
1989  [reg_178]
1990  [channel]
1991  [slot][rank]
1992  [lane].
1993  smallest +
1994  info->
1995  training.
1996  lane_timings
1997  [1][channel]
1998  [slot][rank]
1999  [lane]
2000  -
2001  info->
2002  training.
2003  lane_timings
2004  [0][channel]
2005  [slot][rank]
2006  [lane],
2007  get_timing_register_addr
2008  (lane, 1,
2009  slot, rank),
2010  9, 1);
2011  }
2012  } else
2013  num_successfully_checked[lane]
2014  ++;
2015  }
2016  }
2017  while (!check_bounded(num_successfully_checked, 2))
2018  ;
2019 
2020  for (lane = 0; lane < 8; lane++)
2021  if (state[lane] == COMPLETE) {
2022  write_500(info, channel,
2023  timings[reg_178][channel][slot][rank]
2024  [lane].largest,
2025  get_timing_register_addr(lane, 0,
2026  slot, rank),
2027  9, 1);
2028  write_500(info, channel,
2029  timings[reg_178][channel][slot][rank]
2030  [lane].largest +
2031  info->training.
2032  lane_timings[1][channel][slot][rank]
2033  [lane]
2034  -
2035  info->training.
2036  lane_timings[0][channel][slot][rank]
2037  [lane], get_timing_register_addr(lane,
2038  1,
2039  slot,
2040  rank),
2041  9, 1);
2042  num_successfully_checked[lane] = 0;
2043  } else
2044  num_successfully_checked[lane] = -1;
2045 
2046  do {
2047  int failmask = 0;
2048  for (i = 0; i < niter; i++) {
2049  if (failmask == 0xFF)
2050  break;
2051  failmask |=
2052  check_testing_type2(info, total_rank, 2, i,
2053  0);
2054  failmask |=
2055  check_testing_type2(info, total_rank, 3, i,
2056  1);
2057  }
2058 
2059  mchbar_setbits32(0xfb0, 3 << 16);
2060  for (lane = 0; lane < 8; lane++) {
2061  if (num_successfully_checked[lane] != 0xffff) {
2062  if ((1 << lane) & failmask) {
2063  if (timings[reg_178][channel]
2064  [slot][rank][lane].
2065  largest <=
2066  timings[reg_178][channel]
2067  [slot][rank][lane].
2068  smallest) {
2069  num_successfully_checked
2070  [lane] = -1;
2071  } else {
2072  num_successfully_checked
2073  [lane] = 0;
2074  timings[reg_178]
2075  [channel][slot]
2076  [rank][lane].
2077  largest--;
2078  write_500(info, channel,
2079  timings
2080  [reg_178]
2081  [channel]
2082  [slot][rank]
2083  [lane].
2084  largest,
2085  get_timing_register_addr
2086  (lane, 0,
2087  slot, rank),
2088  9, 1);
2089  write_500(info, channel,
2090  timings
2091  [reg_178]
2092  [channel]
2093  [slot][rank]
2094  [lane].
2095  largest +
2096  info->
2097  training.
2098  lane_timings
2099  [1][channel]
2100  [slot][rank]
2101  [lane]
2102  -
2103  info->
2104  training.
2105  lane_timings
2106  [0][channel]
2107  [slot][rank]
2108  [lane],
2109  get_timing_register_addr
2110  (lane, 1,
2111  slot, rank),
2112  9, 1);
2113  }
2114  } else
2115  num_successfully_checked[lane]
2116  ++;
2117  }
2118  }
2119  }
2120  while (!check_bounded(num_successfully_checked, 3))
2121  ;
2122 
2123  for (lane = 0; lane < 8; lane++) {
2124  write_500(info, channel,
2125  info->training.
2126  lane_timings[0][channel][slot][rank][lane],
2127  get_timing_register_addr(lane, 0, slot, rank),
2128  9, 1);
2129  write_500(info, channel,
2130  info->training.
2131  lane_timings[1][channel][slot][rank][lane],
2132  get_timing_register_addr(lane, 1, slot, rank),
2133  9, 1);
2134  if (timings[reg_178][channel][slot][rank][lane].
2135  largest <=
2136  timings[reg_178][channel][slot][rank][lane].
2137  smallest) {
2138  timings[reg_178][channel][slot][rank][lane].
2139  largest = 0;
2140  timings[reg_178][channel][slot][rank][lane].
2141  smallest = 0;
2142  }
2143  }
2144  }
2145 }
2146 
2147 static void set_10b(struct raminfo *info, u8 val)
2148 {
2149  int channel;
2150  int slot, rank;
2151  int lane;
2152 
2153  if (read_1d0(0x10b, 6) == val)
2154  return;
2155 
2156  write_1d0(val, 0x10b, 6, 1);
2157 
2158  FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 9; lane++) {
2159  u16 reg_500;
2160  reg_500 = read_500(info, channel,
2161  get_timing_register_addr(lane, 0, slot,
2162  rank), 9);
2163  if (val == 1) {
2164  if (lut16[info->clock_speed_index] <= reg_500)
2165  reg_500 -= lut16[info->clock_speed_index];
2166  else
2167  reg_500 = 0;
2168  } else {
2169  reg_500 += lut16[info->clock_speed_index];
2170  }
2171  write_500(info, channel, reg_500,
2172  get_timing_register_addr(lane, 0, slot, rank), 9, 1);
2173  }
2174 }
2175 
2176 static void set_ecc(int onoff)
2177 {
2178  int channel;
2179  for (channel = 0; channel < NUM_CHANNELS; channel++) {
2180  u8 t;
2181  t = mchbar_read8((channel << 10) + 0x5f8);
2182  if (onoff)
2183  t |= 1;
2184  else
2185  t &= ~1;
2186  mchbar_write8((channel << 10) + 0x5f8, t);
2187  }
2188 }
2189 
2190 static void set_178(u8 val)
2191 {
2192  if (val >= 31)
2193  val = val - 31;
2194  else
2195  val = 63 - val;
2196 
2197  write_1d0(2 * val, 0x178, 7, 1);
2198 }
2199 
2200 static void
2201 write_500_timings_type(struct raminfo *info, int channel, int slot, int rank,
2202  int type)
2203 {
2204  int lane;
2205 
2206  for (lane = 0; lane < 8; lane++)
2207  write_500(info, channel,
2208  info->training.
2209  lane_timings[type][channel][slot][rank][lane],
2210  get_timing_register_addr(lane, type, slot, rank), 9,
2211  0);
2212 }
2213 
2214 static void
2215 try_timing_offsets(struct raminfo *info, int channel,
2216  int slot, int rank, int totalrank)
2217 {
2218  u16 count[8];
2219  enum state state[8];
2220  u8 lower_usable[8], upper_usable[8];
2221  int lane;
2222  int i;
2223  int flip = 1;
2224  int timing_offset;
2225 
2226  for (i = 0; i < 8; i++)
2227  state[i] = BEFORE_USABLE;
2228 
2229  memset(count, 0, sizeof(count));
2230 
2231  for (lane = 0; lane < 8; lane++)
2232  write_500(info, channel,
2233  info->training.
2234  lane_timings[2][channel][slot][rank][lane] + 32,
2235  get_timing_register_addr(lane, 3, slot, rank), 9, 1);
2236 
2237  for (timing_offset = 0; !validate_state(state) && timing_offset < 64;
2238  timing_offset++) {
2239  u8 failmask;
2240  write_1d0(timing_offset ^ 32, 0x1bb, 6, 1);
2241  failmask = 0;
2242  for (i = 0; i < 2 && failmask != 0xff; i++) {
2243  flip = !flip;
2244  write_testing(info, totalrank, flip);
2245  failmask |= check_testing(info, totalrank, flip);
2246  }
2247  do_fsm(state, count, failmask, 10, 63, lower_usable,
2248  upper_usable, timing_offset);
2249  }
2250  write_1d0(0, 0x1bb, 6, 1);
2251  dump_timings(info);
2252  if (!validate_state(state))
2253  die("Couldn't discover DRAM timings (1)\n");
2254 
2255  for (lane = 0; lane < 8; lane++) {
2256  u8 bias = 0;
2257 
2258  if (info->silicon_revision) {
2259  int usable_length;
2260 
2261  usable_length = upper_usable[lane] - lower_usable[lane];
2262  if (usable_length >= 20) {
2263  bias = usable_length / 2 - 10;
2264  if (bias >= 2)
2265  bias = 2;
2266  }
2267  }
2268  write_500(info, channel,
2269  info->training.
2270  lane_timings[2][channel][slot][rank][lane] +
2271  (upper_usable[lane] + lower_usable[lane]) / 2 - bias,
2272  get_timing_register_addr(lane, 3, slot, rank), 9, 1);
2273  info->training.timing2_bounds[channel][slot][rank][lane][0] =
2274  info->training.lane_timings[2][channel][slot][rank][lane] +
2275  lower_usable[lane];
2276  info->training.timing2_bounds[channel][slot][rank][lane][1] =
2277  info->training.lane_timings[2][channel][slot][rank][lane] +
2278  upper_usable[lane];
2279  info->training.timing2_offset[channel][slot][rank][lane] =
2280  info->training.lane_timings[2][channel][slot][rank][lane];
2281  }
2282 }
2283 
2284 static u8
2285 choose_training(struct raminfo *info, int channel, int slot, int rank,
2286  int lane, timing_bounds_t * timings, u8 center_178)
2287 {
2288  u16 central_weight;
2289  u16 side_weight;
2290  unsigned int sum = 0, count = 0;
2291  u8 span;
2292  u8 lower_margin, upper_margin;
2293  u8 reg_178;
2294  u8 result;
2295 
2296  span = 12;
2297  central_weight = 20;
2298  side_weight = 20;
2299  if (info->silicon_revision == 1 && channel == 1) {
2300  central_weight = 5;
2301  side_weight = 20;
2302  if ((info->
2303  populated_ranks_mask[1] ^ (info->
2304  populated_ranks_mask[1] >> 2)) &
2305  1)
2306  span = 18;
2307  }
2308  if ((info->populated_ranks_mask[0] & 5) == 5) {
2309  central_weight = 20;
2310  side_weight = 20;
2311  }
2312  if (info->clock_speed_index >= 2
2313  && (info->populated_ranks_mask[0] & 5) == 5 && slot == 1) {
2314  if (info->silicon_revision == 1) {
2315  switch (channel) {
2316  case 0:
2317  if (lane == 1) {
2318  central_weight = 10;
2319  side_weight = 20;
2320  }
2321  break;
2322  case 1:
2323  if (lane == 6) {
2324  side_weight = 5;
2325  central_weight = 20;
2326  }
2327  break;
2328  }
2329  }
2330  if (info->silicon_revision == 0 && channel == 0 && lane == 0) {
2331  side_weight = 5;
2332  central_weight = 20;
2333  }
2334  }
2335  for (reg_178 = center_178 - span; reg_178 <= center_178 + span;
2336  reg_178 += span) {
2337  u8 smallest;
2338  u8 largest;
2339  largest = timings[reg_178][channel][slot][rank][lane].largest;
2340  smallest = timings[reg_178][channel][slot][rank][lane].smallest;
2341  if (largest - smallest + 1 >= 5) {
2342  unsigned int weight;
2343  if (reg_178 == center_178)
2344  weight = central_weight;
2345  else
2346  weight = side_weight;
2347  sum += weight * (largest + smallest);
2348  count += weight;
2349  }
2350  }
2351  dump_timings(info);
2352  if (count == 0)
2353  die("Couldn't discover DRAM timings (2)\n");
2354  result = sum / (2 * count);
2355  lower_margin =
2356  result - timings[center_178][channel][slot][rank][lane].smallest;
2357  upper_margin =
2358  timings[center_178][channel][slot][rank][lane].largest - result;
2359  if (upper_margin < 10 && lower_margin > 10)
2360  result -= MIN(lower_margin - 10, 10 - upper_margin);
2361  if (upper_margin > 10 && lower_margin < 10)
2362  result += MIN(upper_margin - 10, 10 - lower_margin);
2363  return result;
2364 }
2365 
2366 #define STANDARD_MIN_MARGIN 5
2367 
2368 static u8 choose_reg178(struct raminfo *info, timing_bounds_t * timings)
2369 {
2370  u16 margin[64];
2371  int lane, rank, slot, channel;
2372  u8 reg178;
2373  int count = 0, sum = 0;
2374 
2375  for (reg178 = reg178_min[info->clock_speed_index];
2376  reg178 < reg178_max[info->clock_speed_index];
2377  reg178 += reg178_step[info->clock_speed_index]) {
2378  margin[reg178] = -1;
2379  FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 8; lane++) {
2380  int curmargin =
2381  timings[reg178][channel][slot][rank][lane].largest -
2382  timings[reg178][channel][slot][rank][lane].
2383  smallest + 1;
2384  if (curmargin < margin[reg178])
2385  margin[reg178] = curmargin;
2386  }
2387  if (margin[reg178] >= STANDARD_MIN_MARGIN) {
2388  u16 weight;
2389  weight = margin[reg178] - STANDARD_MIN_MARGIN;
2390  sum += weight * reg178;
2391  count += weight;
2392  }
2393  }
2394  dump_timings(info);
2395  if (count == 0)
2396  die("Couldn't discover DRAM timings (3)\n");
2397 
2398  u8 threshold;
2399 
2400  for (threshold = 30; threshold >= 5; threshold--) {
2401  int usable_length = 0;
2402  int smallest_fount = 0;
2403  for (reg178 = reg178_min[info->clock_speed_index];
2404  reg178 < reg178_max[info->clock_speed_index];
2405  reg178 += reg178_step[info->clock_speed_index])
2406  if (margin[reg178] >= threshold) {
2407  usable_length +=
2408  reg178_step[info->clock_speed_index];
2409  info->training.reg178_largest =
2410  reg178 -
2411  2 * reg178_step[info->clock_speed_index];
2412 
2413  if (!smallest_fount) {
2414  smallest_fount = 1;
2415  info->training.reg178_smallest =
2416  reg178 +
2417  reg178_step[info->
2418  clock_speed_index];
2419  }
2420  }
2421  if (usable_length >= 0x21)
2422  break;
2423  }
2424 
2425  return sum / count;
2426 }
2427 
2428 static int check_cached_sanity(struct raminfo *info)
2429 {
2430  int lane;
2431  int slot, rank;
2432  int channel;
2433 
2434  if (!info->cached_training)
2435  return 0;
2436 
2437  for (channel = 0; channel < NUM_CHANNELS; channel++)
2438  for (slot = 0; slot < NUM_SLOTS; slot++)
2439  for (rank = 0; rank < NUM_RANKS; rank++)
2440  for (lane = 0; lane < 8 + info->use_ecc; lane++) {
2441  u16 cached_value, estimation_value;
2442  cached_value =
2443  info->cached_training->
2444  lane_timings[1][channel][slot][rank]
2445  [lane];
2446  if (cached_value >= 0x18
2447  && cached_value <= 0x1E7) {
2448  estimation_value =
2449  info->training.
2450  lane_timings[1][channel]
2451  [slot][rank][lane];
2452  if (estimation_value <
2453  cached_value - 24)
2454  return 0;
2455  if (estimation_value >
2456  cached_value + 24)
2457  return 0;
2458  }
2459  }
2460  return 1;
2461 }
2462 
2463 static int try_cached_training(struct raminfo *info)
2464 {
2465  u8 saved_243[2];
2466  u8 tm;
2467 
2468  int channel, slot, rank, lane;
2469  int flip = 1;
2470  int i, j;
2471 
2472  if (!check_cached_sanity(info))
2473  return 0;
2474 
2475  info->training.reg178_center = info->cached_training->reg178_center;
2476  info->training.reg178_smallest = info->cached_training->reg178_smallest;
2477  info->training.reg178_largest = info->cached_training->reg178_largest;
2478  memcpy(&info->training.timing_bounds,
2479  &info->cached_training->timing_bounds,
2480  sizeof(info->training.timing_bounds));
2481  memcpy(&info->training.timing_offset,
2482  &info->cached_training->timing_offset,
2483  sizeof(info->training.timing_offset));
2484 
2485  write_1d0(2, 0x142, 3, 1);
2486  saved_243[0] = mchbar_read8(0x243);
2487  saved_243[1] = mchbar_read8(0x643);
2488  mchbar_write8(0x243, saved_243[0] | 2);
2489  mchbar_write8(0x643, saved_243[1] | 2);
2490  set_ecc(0);
2491  pci_write_config16(NORTHBRIDGE, 0xc8, 3);
2492  if (read_1d0(0x10b, 6) & 1)
2493  set_10b(info, 0);
2494  for (tm = 0; tm < 2; tm++) {
2495  int totalrank;
2496 
2497  set_178(tm ? info->cached_training->reg178_largest : info->
2498  cached_training->reg178_smallest);
2499 
2500  totalrank = 0;
2501  /* Check timing ranges. With i == 0 we check smallest one and with
2502  i == 1 the largest bound. With j == 0 we check that on the bound
2503  it still works whereas with j == 1 we check that just outside of
2504  bound we fail.
2505  */
2506  FOR_POPULATED_RANKS_BACKWARDS {
2507  for (i = 0; i < 2; i++) {
2508  for (lane = 0; lane < 8; lane++) {
2509  write_500(info, channel,
2510  info->cached_training->
2511  timing2_bounds[channel][slot]
2512  [rank][lane][i],
2513  get_timing_register_addr(lane,
2514  3,
2515  slot,
2516  rank),
2517  9, 1);
2518 
2519  if (!i)
2520  write_500(info, channel,
2521  info->
2522  cached_training->
2523  timing2_offset
2524  [channel][slot][rank]
2525  [lane],
2526  get_timing_register_addr
2527  (lane, 2, slot, rank),
2528  9, 1);
2529  write_500(info, channel,
2530  i ? info->cached_training->
2531  timing_bounds[tm][channel]
2532  [slot][rank][lane].
2533  largest : info->
2534  cached_training->
2535  timing_bounds[tm][channel]
2536  [slot][rank][lane].smallest,
2537  get_timing_register_addr(lane,
2538  0,
2539  slot,
2540  rank),
2541  9, 1);
2542  write_500(info, channel,
2543  info->cached_training->
2544  timing_offset[channel][slot]
2545  [rank][lane] +
2546  (i ? info->cached_training->
2547  timing_bounds[tm][channel]
2548  [slot][rank][lane].
2549  largest : info->
2550  cached_training->
2551  timing_bounds[tm][channel]
2552  [slot][rank][lane].
2553  smallest) - 64,
2554  get_timing_register_addr(lane,
2555  1,
2556  slot,
2557  rank),
2558  9, 1);
2559  }
2560  for (j = 0; j < 2; j++) {
2561  u8 failmask;
2562  u8 expected_failmask;
2563  char reg1b3;
2564 
2565  reg1b3 = (j == 1) + 4;
2566  reg1b3 =
2567  j == i ? reg1b3 : (-reg1b3) & 0x3f;
2568  write_1d0(reg1b3, 0x1bb, 6, 1);
2569  write_1d0(reg1b3, 0x1b3, 6, 1);
2570  write_1d0(reg1b3, 0x1a3, 6, 1);
2571 
2572  flip = !flip;
2573  write_testing(info, totalrank, flip);
2574  failmask =
2575  check_testing(info, totalrank,
2576  flip);
2577  expected_failmask =
2578  j == 0 ? 0x00 : 0xff;
2579  if (failmask != expected_failmask)
2580  goto fail;
2581  }
2582  }
2583  totalrank++;
2584  }
2585  }
2586 
2587  set_178(info->cached_training->reg178_center);
2588  if (info->use_ecc)
2589  set_ecc(1);
2590  write_training_data(info);
2591  write_1d0(0, 322, 3, 1);
2592  info->training = *info->cached_training;
2593 
2594  write_1d0(0, 0x1bb, 6, 1);
2595  write_1d0(0, 0x1b3, 6, 1);
2596  write_1d0(0, 0x1a3, 6, 1);
2597  mchbar_write8(0x243, saved_243[0]);
2598  mchbar_write8(0x643, saved_243[1]);
2599 
2600  return 1;
2601 
2602 fail:
2603  FOR_POPULATED_RANKS {
2604  write_500_timings_type(info, channel, slot, rank, 1);
2605  write_500_timings_type(info, channel, slot, rank, 2);
2606  write_500_timings_type(info, channel, slot, rank, 3);
2607  }
2608 
2609  write_1d0(0, 0x1bb, 6, 1);
2610  write_1d0(0, 0x1b3, 6, 1);
2611  write_1d0(0, 0x1a3, 6, 1);
2612  mchbar_write8(0x243, saved_243[0]);
2613  mchbar_write8(0x643, saved_243[1]);
2614 
2615  return 0;
2616 }
2617 
2618 static void do_ram_training(struct raminfo *info)
2619 {
2620  u8 saved_243[2];
2621  int totalrank = 0;
2622  u8 reg_178;
2623  int niter;
2624 
2625  timing_bounds_t *timings = timings_car;
2626  int lane, rank, slot, channel;
2627  u8 reg178_center;
2628 
2629  write_1d0(2, 0x142, 3, 1);
2630  saved_243[0] = mchbar_read8(0x243);
2631  saved_243[1] = mchbar_read8(0x643);
2632  mchbar_write8(0x243, saved_243[0] | 2);
2633  mchbar_write8(0x643, saved_243[1] | 2);
2634  switch (info->clock_speed_index) {
2635  case 0:
2636  niter = 5;
2637  break;
2638  case 1:
2639  niter = 10;
2640  break;
2641  default:
2642  niter = 19;
2643  break;
2644  }
2645  set_ecc(0);
2646 
2647  FOR_POPULATED_RANKS_BACKWARDS {
2648  int i;
2649 
2650  write_500_timings_type(info, channel, slot, rank, 0);
2651 
2652  write_testing(info, totalrank, 0);
2653  for (i = 0; i < niter; i++) {
2654  write_testing_type2(info, totalrank, 2, i, 0);
2655  write_testing_type2(info, totalrank, 3, i, 1);
2656  }
2657  pci_write_config8(NORTHBRIDGE, 0xc0, 0x01);
2658  totalrank++;
2659  }
2660 
2661  if (reg178_min[info->clock_speed_index] <
2662  reg178_max[info->clock_speed_index])
2663  memset(timings[reg178_min[info->clock_speed_index]], 0,
2664  sizeof(timings[0]) *
2665  (reg178_max[info->clock_speed_index] -
2666  reg178_min[info->clock_speed_index]));
2667  for (reg_178 = reg178_min[info->clock_speed_index];
2668  reg_178 < reg178_max[info->clock_speed_index];
2669  reg_178 += reg178_step[info->clock_speed_index]) {
2670  totalrank = 0;
2671  set_178(reg_178);
2672  for (channel = NUM_CHANNELS - 1; channel >= 0; channel--)
2673  for (slot = 0; slot < NUM_SLOTS; slot++)
2674  for (rank = 0; rank < NUM_RANKS; rank++) {
2675  memset(&timings[reg_178][channel][slot]
2676  [rank][0].smallest, 0, 16);
2677  if (info->
2678  populated_ranks[channel][slot]
2679  [rank]) {
2680  train_ram_at_178(info, channel,
2681  slot, rank,
2682  totalrank,
2683  reg_178, 1,
2684  niter,
2685  timings);
2686  totalrank++;
2687  }
2688  }
2689  }
2690 
2691  reg178_center = choose_reg178(info, timings);
2692 
2693  FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 8; lane++) {
2694  info->training.timing_bounds[0][channel][slot][rank][lane].
2695  smallest =
2696  timings[info->training.
2697  reg178_smallest][channel][slot][rank][lane].
2698  smallest;
2699  info->training.timing_bounds[0][channel][slot][rank][lane].
2700  largest =
2701  timings[info->training.
2702  reg178_smallest][channel][slot][rank][lane].largest;
2703  info->training.timing_bounds[1][channel][slot][rank][lane].
2704  smallest =
2705  timings[info->training.
2706  reg178_largest][channel][slot][rank][lane].smallest;
2707  info->training.timing_bounds[1][channel][slot][rank][lane].
2708  largest =
2709  timings[info->training.
2710  reg178_largest][channel][slot][rank][lane].largest;
2711  info->training.timing_offset[channel][slot][rank][lane] =
2712  info->training.lane_timings[1][channel][slot][rank][lane]
2713  -
2714  info->training.lane_timings[0][channel][slot][rank][lane] +
2715  64;
2716  }
2717 
2718  if (info->silicon_revision == 1
2719  && (info->
2720  populated_ranks_mask[1] ^ (info->
2721  populated_ranks_mask[1] >> 2)) & 1) {
2722  int ranks_after_channel1;
2723 
2724  totalrank = 0;
2725  for (reg_178 = reg178_center - 18;
2726  reg_178 <= reg178_center + 18; reg_178 += 18) {
2727  totalrank = 0;
2728  set_178(reg_178);
2729  for (slot = 0; slot < NUM_SLOTS; slot++)
2730  for (rank = 0; rank < NUM_RANKS; rank++) {
2731  if (info->
2732  populated_ranks[1][slot][rank]) {
2733  train_ram_at_178(info, 1, slot,
2734  rank,
2735  totalrank,
2736  reg_178, 0,
2737  niter,
2738  timings);
2739  totalrank++;
2740  }
2741  }
2742  }
2743  ranks_after_channel1 = totalrank;
2744 
2745  for (reg_178 = reg178_center - 12;
2746  reg_178 <= reg178_center + 12; reg_178 += 12) {
2747  totalrank = ranks_after_channel1;
2748  set_178(reg_178);
2749  for (slot = 0; slot < NUM_SLOTS; slot++)
2750  for (rank = 0; rank < NUM_RANKS; rank++)
2751  if (info->
2752  populated_ranks[0][slot][rank]) {
2753  train_ram_at_178(info, 0, slot,
2754  rank,
2755  totalrank,
2756  reg_178, 0,
2757  niter,
2758  timings);
2759  totalrank++;
2760  }
2761 
2762  }
2763  } else {
2764  for (reg_178 = reg178_center - 12;
2765  reg_178 <= reg178_center + 12; reg_178 += 12) {
2766  totalrank = 0;
2767  set_178(reg_178);
2768  FOR_POPULATED_RANKS_BACKWARDS {
2769  train_ram_at_178(info, channel, slot, rank,
2770  totalrank, reg_178, 0, niter,
2771  timings);
2772  totalrank++;
2773  }
2774  }
2775  }
2776 
2777  set_178(reg178_center);
2778  FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 8; lane++) {
2779  u16 tm0;
2780 
2781  tm0 =
2782  choose_training(info, channel, slot, rank, lane, timings,
2783  reg178_center);
2784  write_500(info, channel, tm0,
2785  get_timing_register_addr(lane, 0, slot, rank), 9, 1);
2786  write_500(info, channel,
2787  tm0 +
2788  info->training.
2789  lane_timings[1][channel][slot][rank][lane] -
2790  info->training.
2791  lane_timings[0][channel][slot][rank][lane],
2792  get_timing_register_addr(lane, 1, slot, rank), 9, 1);
2793  }
2794 
2795  totalrank = 0;
2796  FOR_POPULATED_RANKS_BACKWARDS {
2797  try_timing_offsets(info, channel, slot, rank, totalrank);
2798  totalrank++;
2799  }
2800  mchbar_write8(0x243, saved_243[0]);
2801  mchbar_write8(0x643, saved_243[1]);
2802  write_1d0(0, 0x142, 3, 1);
2803  info->training.reg178_center = reg178_center;
2804 }
2805 
2806 static void ram_training(struct raminfo *info)
2807 {
2808  u16 saved_fc4;
2809 
2810  saved_fc4 = mchbar_read16(0xfc4);
2811  mchbar_write16(0xfc4, 0xffff);
2812 
2813  if (info->revision >= 8)
2814  read_4090(info);
2815 
2816  if (!try_cached_training(info))
2817  do_ram_training(info);
2818  if ((info->silicon_revision == 2 || info->silicon_revision == 3)
2819  && info->clock_speed_index < 2)
2820  set_10b(info, 1);
2821  mchbar_write16(0xfc4, saved_fc4);
2822 }
2823 
2824 u16 get_max_timing(struct raminfo *info, int channel)
2825 {
2826  int slot, rank, lane;
2827  u16 ret = 0;
2828 
2829  if ((mchbar_read8(0x2ca8) >> 2) < 1)
2830  return 384;
2831 
2832  if (info->revision < 8)
2833  return 256;
2834 
2835  for (slot = 0; slot < NUM_SLOTS; slot++)
2836  for (rank = 0; rank < NUM_RANKS; rank++)
2837  if (info->populated_ranks[channel][slot][rank])
2838  for (lane = 0; lane < 8 + info->use_ecc; lane++)
2839  ret = MAX(ret, read_500(info, channel,
2840  get_timing_register_addr
2841  (lane, 0, slot,
2842  rank), 9));
2843  return ret;
2844 }
2845 
2846 static void dmi_setup(void)
2847 {
2848  gav(dmibar_read8(0x254));
2849  dmibar_write8(0x254, 1 << 0);
2850  dmibar_write16(0x1b8, 0x18f2);
2851  mchbar_clrsetbits16(0x48, ~0, 1 << 1);
2852 
2853  dmibar_setbits32(0xd68, 1 << 27);
2854 
2855  outl((gav(inl(DEFAULT_GPIOBASE | 0x38)) & ~0x140000) | 0x400000,
2856  DEFAULT_GPIOBASE | 0x38);
2857  gav(inb(DEFAULT_GPIOBASE | 0xe)); // = 0xfdcaff6e
2858 }
2859 
2860 void chipset_init(const int s3resume)
2861 {
2862  u8 x2ca8;
2863  u16 ggc;
2864  u8 gfxsize;
2865 
2866  x2ca8 = mchbar_read8(0x2ca8);
2867  if ((x2ca8 & 1) || (x2ca8 == 8 && !s3resume)) {
2868  printk(BIOS_DEBUG, "soft reset detected, rebooting properly\n");
2869  mchbar_write8(0x2ca8, 0);
2870  system_reset();
2871  }
2872 
2873  dmi_setup();
2874 
2875  mchbar_write16(0x1170, 0xa880);
2876  mchbar_write8(0x11c1, 1 << 0);
2877  mchbar_write16(0x1170, 0xb880);
2878  mchbar_clrsetbits8(0x1210, ~0, 0x84);
2879 
2880  gfxsize = get_uint_option("gfx_uma_size", 0); /* 0 for 32MB */
2881 
2882  ggc = 0xb00 | ((gfxsize + 5) << 4);
2883 
2884  pci_write_config16(NORTHBRIDGE, GGC, ggc | 2);
2885 
2886  u16 deven;
2887  deven = pci_read_config16(NORTHBRIDGE, DEVEN); // = 0x3
2888 
2889  if (deven & 8) {
2890  mchbar_write8(0x2c30, 1 << 5);
2891  pci_read_config8(NORTHBRIDGE, 0x8); // = 0x18
2892  mchbar_setbits16(0x2c30, 1 << 9);
2893  mchbar_write16(0x2c32, 0x434);
2894  mchbar_clrsetbits32(0x2c44, ~0, 0x1053687);
2895  pci_read_config8(GMA, MSAC); // = 0x2
2896  pci_write_config8(GMA, MSAC, 0x2);
2897  RCBA8(0x2318);
2898  RCBA8(0x2318) = 0x47;
2899  RCBA8(0x2320);
2900  RCBA8(0x2320) = 0xfc;
2901  }
2902 
2903  mchbar_clrsetbits32(0x30, ~0, 0x40);
2904 
2905  pci_write_config16(NORTHBRIDGE, GGC, ggc);
2906  gav(RCBA32(0x3428));
2907  RCBA32(0x3428) = 0x1d;
2908 }
2909 
2910 static u8 get_bits_420(const u32 reg32)
2911 {
2912  u8 val = 0;
2913  val |= (reg32 >> 4) & (1 << 0);
2914  val |= (reg32 >> 2) & (1 << 1);
2915  val |= (reg32 >> 0) & (1 << 2);
2916  return val;
2917 }
2918 
2919 void raminit(const int s3resume, const u8 *spd_addrmap)
2920 {
2921  unsigned int channel, slot, lane, rank;
2922  struct raminfo info;
2923  u8 x2ca8;
2924  int cbmem_wasnot_inited;
2925 
2926  x2ca8 = mchbar_read8(0x2ca8);
2927 
2928  printk(RAM_DEBUG, "Scratchpad MCHBAR8(0x2ca8): 0x%04x\n", x2ca8);
2929 
2930  memset(&info, 0x5a, sizeof(info));
2931 
2932  info.last_500_command[0] = 0;
2933  info.last_500_command[1] = 0;
2934 
2935  info.board_lane_delay[0] = 0x14;
2936  info.board_lane_delay[1] = 0x07;
2937  info.board_lane_delay[2] = 0x07;
2938  info.board_lane_delay[3] = 0x08;
2939  info.board_lane_delay[4] = 0x56;
2940  info.board_lane_delay[5] = 0x04;
2941  info.board_lane_delay[6] = 0x04;
2942  info.board_lane_delay[7] = 0x05;
2943  info.board_lane_delay[8] = 0x10;
2944 
2945  info.training.reg_178 = 0;
2946  info.training.reg_10b = 0;
2947 
2948  /* Wait for some bit, maybe TXT clear. */
2949  while (!(read8((u8 *)0xfed40000) & (1 << 7)))
2950  ;
2951 
2952  /* Wait for ME to be ready */
2953  intel_early_me_init();
2954  info.memory_reserved_for_heci_mb = intel_early_me_uma_size();
2955 
2956  /* before SPD */
2957  timestamp_add_now(101);
2958 
2959  if (!s3resume || 1) { // possible error
2960  memset(&info.populated_ranks, 0, sizeof(info.populated_ranks));
2961 
2962  info.use_ecc = 1;
2963  for (channel = 0; channel < NUM_CHANNELS; channel++)
2964  for (slot = 0; slot < NUM_SLOTS; slot++) {
2965  int v;
2966  int try;
2967  int addr;
2968  const u8 useful_addresses[] = {
2969  DEVICE_TYPE,
2970  MODULE_TYPE,
2971  DENSITY,
2972  RANKS_AND_DQ,
2973  MEMORY_BUS_WIDTH,
2974  TIMEBASE_DIVIDEND,
2975  TIMEBASE_DIVISOR,
2976  CYCLETIME,
2977  CAS_LATENCIES_LSB,
2978  CAS_LATENCIES_MSB,
2979  CAS_LATENCY_TIME,
2980  0x11, 0x12, 0x13, 0x14, 0x15,
2981  0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b,
2982  0x1c, 0x1d,
2983  THERMAL_AND_REFRESH,
2984  0x20,
2985  REFERENCE_RAW_CARD_USED,
2986  RANK1_ADDRESS_MAPPING,
2987  0x75, 0x76, 0x77, 0x78,
2988  0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e,
2989  0x7f, 0x80, 0x81, 0x82, 0x83, 0x84,
2990  0x85, 0x86, 0x87, 0x88,
2991  0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e,
2992  0x8f, 0x90, 0x91, 0x92, 0x93, 0x94,
2993  0x95
2994  };
2995  if (!spd_addrmap[2 * channel + slot])
2996  continue;
2997  for (try = 0; try < 5; try++) {
2998  v = smbus_read_byte(spd_addrmap[2 * channel + slot],
2999  DEVICE_TYPE);
3000  if (v >= 0)
3001  break;
3002  }
3003  if (v < 0)
3004  continue;
3005  for (addr = 0;
3006  addr <
3007  ARRAY_SIZE(useful_addresses); addr++)
3008  gav(info.
3009  spd[channel][0][useful_addresses
3010  [addr]] =
3011  smbus_read_byte(spd_addrmap[2 * channel + slot],
3012  useful_addresses
3013  [addr]));
3014  if (info.spd[channel][0][DEVICE_TYPE] != 11)
3015  die("Only DDR3 is supported");
3016 
3017  v = info.spd[channel][0][RANKS_AND_DQ];
3018  info.populated_ranks[channel][0][0] = 1;
3019  info.populated_ranks[channel][0][1] =
3020  ((v >> 3) & 7);
3021  if (((v >> 3) & 7) > 1)
3022  die("At most 2 ranks are supported");
3023  if ((v & 7) == 0 || (v & 7) > 2)
3024  die("Only x8 and x16 modules are supported");
3025  if ((info.
3026  spd[channel][slot][MODULE_TYPE] & 0xF) != 2
3027  && (info.
3028  spd[channel][slot][MODULE_TYPE] & 0xF)
3029  != 3)
3030  die("Registered memory is not supported");
3031  info.is_x16_module[channel][0] = (v & 7) - 1;
3032  info.density[channel][slot] =
3033  info.spd[channel][slot][DENSITY] & 0xF;
3034  if (!
3035  (info.
3036  spd[channel][slot][MEMORY_BUS_WIDTH] &
3037  0x18))
3038  info.use_ecc = 0;
3039  }
3040 
3041  gav(0x55);
3042 
3043  for (channel = 0; channel < NUM_CHANNELS; channel++) {
3044  int v = 0;
3045  for (slot = 0; slot < NUM_SLOTS; slot++)
3046  for (rank = 0; rank < NUM_RANKS; rank++)
3047  v |= info.
3048  populated_ranks[channel][slot][rank]
3049  << (2 * slot + rank);
3050  info.populated_ranks_mask[channel] = v;
3051  }
3052 
3053  gav(0x55);
3054 
3055  gav(pci_read_config32(NORTHBRIDGE, CAPID0 + 4));
3056  }
3057 
3058  /* after SPD */
3059  timestamp_add_now(102);
3060 
3061  mchbar_clrbits8(0x2ca8, 1 << 1 | 1 << 0);
3062 
3063  collect_system_info(&info);
3064  calculate_timings(&info);
3065 
3066  if (!s3resume) {
3067  u8 reg8 = pci_read_config8(SOUTHBRIDGE, GEN_PMCON_2);
3068  if (x2ca8 == 0 && (reg8 & 0x80)) {
3069  /* Don't enable S4-assertion stretch. Makes trouble on roda/rk9.
3070  reg8 = pci_read_config8(PCI_DEV(0, 0x1f, 0), 0xa4);
3071  pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa4, reg8 | 0x08);
3072  */
3073 
3074  /* Clear bit7. */
3075 
3076  pci_write_config8(SOUTHBRIDGE, GEN_PMCON_2,
3077  (reg8 & ~(1 << 7)));
3078 
3079  printk(BIOS_INFO,
3080  "Interrupted RAM init, reset required.\n");
3081  system_reset();
3082  }
3083  }
3084 
3085  if (!s3resume && x2ca8 == 0)
3086  pci_write_config8(SOUTHBRIDGE, GEN_PMCON_2,
3087  pci_read_config8(SOUTHBRIDGE, GEN_PMCON_2) | 0x80);
3088 
3089  compute_derived_timings(&info);
3090 
3091  early_quickpath_init(&info, x2ca8);
3092 
3093  info.cached_training = get_cached_training();
3094 
3095  if (x2ca8 == 0)
3096  late_quickpath_init(&info, s3resume);
3097 
3098  mchbar_setbits32(0x2c80, 1 << 24);
3099  mchbar_write32(0x1804, mchbar_read32(0x1c04) & ~(1 << 27));
3100 
3101  mchbar_read8(0x2ca8); // !!!!
3102 
3103  if (x2ca8 == 0) {
3104  mchbar_clrbits8(0x2ca8, 3);
3105  mchbar_write8(0x2ca8, mchbar_read8(0x2ca8) + 4); // "+" or "|"?
3106  /* This issues a CPU reset without resetting the platform */
3107  printk(BIOS_DEBUG, "Issuing a CPU reset\n");
3108  /* Write back the S3 state to PM1_CNT to let the reset CPU
3109  know it also needs to take the s3 path. */
3110  if (s3resume)
3111  write_pmbase32(PM1_CNT, read_pmbase32(PM1_CNT)
3112  | (SLP_TYP_S3 << 10));
3113  mchbar_setbits32(0x1af0, 1 << 4);
3114  halt();
3115  }
3116 
3117  mchbar_clrbits8(0x2ca8, 0); // !!!!
3118 
3119  mchbar_clrbits32(0x2c80, 1 << 24);
3120 
3121  pci_write_config32(QPI_NON_CORE, MAX_RTIDS, 0x20220);
3122 
3123  {
3124  u8 x2c20 = (mchbar_read16(0x2c20) >> 8) & 3;
3125  u16 x2c10 = mchbar_read16(0x2c10);
3126  u16 value = mchbar_read16(0x2c00);
3127  if (x2c20 == 0 && (x2c10 & 0x300) == 0)
3128  value |= (1 << 7);
3129  else
3130  value &= ~(1 << 0);
3131 
3132  mchbar_write16(0x2c00, value);
3133  }
3134 
3135  udelay(1000); // !!!!
3136 
3137  write_1d0(0, 0x33d, 0, 0);
3138  write_500(&info, 0, 0, 0xb61, 0, 0);
3139  write_500(&info, 1, 0, 0xb61, 0, 0);
3140  mchbar_write32(0x1a30, 0);
3141  mchbar_write32(0x1a34, 0);
3142  mchbar_write16(0x614, 0xb5b | (info.populated_ranks[1][0][0] * 0x404) |
3143  (info.populated_ranks[0][0][0] * 0xa0));
3144  mchbar_write16(0x616, 0x26a);
3145  mchbar_write32(0x134, 0x856000);
3146  mchbar_write32(0x160, 0x5ffffff);
3147  mchbar_clrsetbits32(0x114, ~0, 0xc2024440); // !!!!
3148  mchbar_clrsetbits32(0x118, ~0, 0x4); // !!!!
3149  for (channel = 0; channel < NUM_CHANNELS; channel++)
3150  mchbar_write32(0x260 + (channel << 10), 0x30809ff |
3151  (info.populated_ranks_mask[channel] & 3) << 20);
3152  for (channel = 0; channel < NUM_CHANNELS; channel++) {
3153  mchbar_write16(0x31c + (channel << 10), 0x101);
3154  mchbar_write16(0x360 + (channel << 10), 0x909);
3155  mchbar_write16(0x3a4 + (channel << 10), 0x101);
3156  mchbar_write16(0x3e8 + (channel << 10), 0x101);
3157  mchbar_write32(0x320 + (channel << 10), 0x29002900);
3158  mchbar_write32(0x324 + (channel << 10), 0);
3159  mchbar_write32(0x368 + (channel << 10), 0x32003200);
3160  mchbar_write16(0x352 + (channel << 10), 0x505);
3161  mchbar_write16(0x354 + (channel << 10), 0x3c3c);
3162  mchbar_write16(0x356 + (channel << 10), 0x1040);
3163  mchbar_write16(0x39a + (channel << 10), 0x73e4);
3164  mchbar_write16(0x3de + (channel << 10), 0x77ed);
3165  mchbar_write16(0x422 + (channel << 10), 0x1040);
3166  }
3167 
3168  write_1d0(0x4, 0x151, 4, 1);
3169  write_1d0(0, 0x142, 3, 1);
3170  rdmsr(0x1ac); // !!!!
3171  write_500(&info, 1, 1, 0x6b3, 4, 1);
3172  write_500(&info, 1, 1, 0x6cf, 4, 1);
3173 
3174  rmw_1d0(0x21c, 0x38, 0, 6);
3175 
3176  write_1d0(((!info.populated_ranks[1][0][0]) << 1) | ((!info.
3177  populated_ranks[0]
3178  [0][0]) << 0),
3179  0x1d1, 3, 1);
3180  for (channel = 0; channel < NUM_CHANNELS; channel++) {
3181  mchbar_write16(0x38e + (channel << 10), 0x5f5f);
3182  mchbar_write16(0x3d2 + (channel << 10), 0x5f5f);
3183  }
3184 
3185  set_334(0);
3186 
3187  program_base_timings(&info);
3188 
3189  mchbar_setbits8(0x5ff, 1 << 7);
3190 
3191  write_1d0(0x2, 0x1d5, 2, 1);
3192  write_1d0(0x20, 0x166, 7, 1);
3193  write_1d0(0x0, 0xeb, 3, 1);
3194  write_1d0(0x0, 0xf3, 6, 1);
3195 
3196  for (channel = 0; channel < NUM_CHANNELS; channel++) {
3197  u8 a = 0;
3198  if (info.populated_ranks[channel][0][1] && info.clock_speed_index > 1)
3199  a = 3;
3200  if (info.silicon_revision == 0 || info.silicon_revision == 1)
3201  a = 3;
3202 
3203  for (lane = 0; lane < 9; lane++) {
3204  const u16 addr = 0x125 + get_lane_offset(0, 0, lane);
3205  rmw_500(&info, channel, addr, 6, 0xf, a);
3206  }
3207  }
3208 
3209  if (s3resume) {
3210  if (info.cached_training == NULL) {
3211  u32 reg32;
3212  printk(BIOS_ERR,
3213  "Couldn't find training data. Rebooting\n");
3214  reg32 = inl(DEFAULT_PMBASE + 0x04);
3215  outl(reg32 & ~(7 << 10), DEFAULT_PMBASE + 0x04);
3216  full_reset();
3217  }
3218  int tm;
3219  info.training = *info.cached_training;
3220  for (tm = 0; tm < 4; tm++)
3221  for (channel = 0; channel < NUM_CHANNELS; channel++)
3222  for (slot = 0; slot < NUM_SLOTS; slot++)
3223  for (rank = 0; rank < NUM_RANKS; rank++)
3224  for (lane = 0; lane < 9; lane++)
3225  write_500(&info,
3226  channel,
3227  info.training.
3228  lane_timings
3229  [tm][channel]
3230  [slot][rank]
3231  [lane],
3232  get_timing_register_addr
3233  (lane, tm,
3234  slot, rank),
3235  9, 0);
3236  write_1d0(info.cached_training->reg_178, 0x178, 7, 1);
3237  write_1d0(info.cached_training->reg_10b, 0x10b, 6, 1);
3238  }
3239 
3240  mchbar_clrsetbits32(0x1f4, ~0, 1 << 17); // !!!!
3241  mchbar_write32(0x1f0, 0x1d000200);
3242  mchbar_setbits8(0x1f0, 1 << 0);
3243  while (mchbar_read8(0x1f0) & 1)
3244  ;
3245 
3246  program_board_delay(&info);
3247 
3248  mchbar_write8(0x5ff, 0);
3249  mchbar_write8(0x5ff, 1 << 7);
3250  mchbar_write8(0x5f4, 1 << 0);
3251 
3252  mchbar_clrbits32(0x130, 1 << 1); // | 2 when ?
3253  while (mchbar_read32(0x130) & 1)
3254  ;
3255 
3256  rmw_1d0(0x14b, 0x47, 0x30, 7);
3257  rmw_1d0(0xd6, 0x38, 7, 6);
3258  rmw_1d0(0x328, 0x38, 7, 6);
3259 
3260  for (channel = 0; channel < NUM_CHANNELS; channel++)
3261  set_4cf(&info, channel, 1, 0);
3262 
3263  rmw_1d0(0x116, 0xe, 0, 4);
3264  rmw_1d0(0xae, 0x3e, 0, 6);
3265  rmw_1d0(0x300, 0x3e, 0, 6);
3266  mchbar_clrbits16(0x356, 1 << 15);
3267  mchbar_clrbits16(0x756, 1 << 15);
3268  mchbar_clrbits32(0x140, 7 << 24);
3269  mchbar_clrbits32(0x138, 7 << 24);
3270  mchbar_write32(0x130, 0x31111301);
3271  /* Wait until REG130b0 is 1. */
3272  while (mchbar_read32(0x130) & 1)
3273  ;
3274 
3275  u8 value_a1;
3276  {
3277  const u8 val_xa1 = get_bits_420(read_1d0(0xa1, 6)); // = 0x1cf4040 // !!!!
3278  const u8 val_2f3 = get_bits_420(read_1d0(0x2f3, 6)); // = 0x10a4040 // !!!!
3279  value_a1 = val_xa1;
3280  rmw_1d0(0x320, 0x38, val_2f3, 6);
3281  rmw_1d0(0x14b, 0x78, val_xa1, 7);
3282  rmw_1d0(0xce, 0x38, val_xa1, 6);
3283  }
3284 
3285  for (channel = 0; channel < NUM_CHANNELS; channel++)
3286  set_4cf(&info, channel, 1, 1);
3287 
3288  rmw_1d0(0x116, 0xe, 1, 4); // = 0x4040432 // !!!!
3289  {
3290  if ((mchbar_read32(0x144) & 0x1f) < 0x13)
3291  value_a1 += 2;
3292  else
3293  value_a1 += 1;
3294 
3295  if (value_a1 > 7)
3296  value_a1 = 7;
3297 
3298  write_1d0(2, 0xae, 6, 1);
3299  write_1d0(2, 0x300, 6, 1);
3300  write_1d0(value_a1, 0x121, 3, 1);
3301  rmw_1d0(0xd6, 0x38, 4, 6);
3302  rmw_1d0(0x328, 0x38, 4, 6);
3303  }
3304 
3305  for (channel = 0; channel < NUM_CHANNELS; channel++)
3306  set_4cf(&info, channel, 2, 0);
3307 
3308  mchbar_write32(0x130, 0x11111301 | info.populated_ranks[1][0][0] << 30 |
3309  info.populated_ranks[0][0][0] << 29);
3310  while (mchbar_read8(0x130) & 1)
3311  ;
3312 
3313  {
3314  const u8 val_xa1 = get_bits_420(read_1d0(0xa1, 6));
3315  read_1d0(0x2f3, 6); // = 0x10a4054 // !!!!
3316  rmw_1d0(0x21c, 0x38, 0, 6);
3317  rmw_1d0(0x14b, 0x78, val_xa1, 7);
3318  }
3319 
3320  for (channel = 0; channel < NUM_CHANNELS; channel++)
3321  set_4cf(&info, channel, 2, 1);
3322 
3323  set_334(1);
3324 
3325  mchbar_write8(0x1e8, 1 << 2);
3326 
3327  for (channel = 0; channel < NUM_CHANNELS; channel++) {
3328  write_500(&info, channel,
3329  0x3 & ~(info.populated_ranks_mask[channel]), 0x6b7, 2,
3330  1);
3331  write_500(&info, channel, 0x3, 0x69b, 2, 1);
3332  }
3333  mchbar_clrsetbits32(0x2d0, ~0xff0c01ff, 0x200000);
3334  mchbar_write16(0x6c0, 0x14a0);
3335  mchbar_clrsetbits32(0x6d0, ~0xff0000ff, 0x8000);
3336  mchbar_write16(0x232, 1 << 3);
3337  /* 0x40004 or 0 depending on ? */
3338  mchbar_clrsetbits32(0x234, 0x40004, 0x40004);
3339  mchbar_clrsetbits32(0x34, 0x7, 5);
3340  mchbar_write32(0x128, 0x2150d05);
3341  mchbar_write8(0x12c, 0x1f);
3342  mchbar_write8(0x12d, 0x56);
3343  mchbar_write8(0x12e, 0x31);
3344  mchbar_write8(0x12f, 0);
3345  mchbar_write8(0x271, 1 << 1);
3346  mchbar_write8(0x671, 1 << 1);
3347  mchbar_write8(0x1e8, 1 << 2);
3348  for (channel = 0; channel < NUM_CHANNELS; channel++)
3349  mchbar_write32(0x294 + (channel << 10),
3350  (info.populated_ranks_mask[channel] & 3) << 16);
3351  mchbar_clrsetbits32(0x134, ~0xfc01ffff, 0x10000);
3352  mchbar_clrsetbits32(0x134, ~0xfc85ffff, 0x850000);
3353  for (channel = 0; channel < NUM_CHANNELS; channel++)
3354  mchbar_clrsetbits32(0x260 + (channel << 10), 0xf << 20, 1 << 27 |
3355  (info.populated_ranks_mask[channel] & 3) << 20);
3356 
3357  if (!s3resume)
3358  jedec_init(&info);
3359 
3360  int totalrank = 0;
3361  for (channel = 0; channel < NUM_CHANNELS; channel++)
3362  for (slot = 0; slot < NUM_SLOTS; slot++)
3363  for (rank = 0; rank < NUM_RANKS; rank++)
3364  if (info.populated_ranks[channel][slot][rank]) {
3365  jedec_read(&info, channel, slot, rank,
3366  totalrank, 0xa, 0x400);
3367  totalrank++;
3368  }
3369 
3370  mchbar_write8(0x12c, 0x9f);
3371 
3372  mchbar_clrsetbits8(0x271, 0x3e, 0x0e);
3373  mchbar_clrsetbits8(0x671, 0x3e, 0x0e);
3374 
3375  if (!s3resume) {
3376  for (channel = 0; channel < NUM_CHANNELS; channel++) {
3377  mchbar_write32(0x294 + (channel << 10),
3378  (info.populated_ranks_mask[channel] & 3) << 16);
3379  mchbar_write16(0x298 + (channel << 10),
3380  info.populated_ranks[channel][0][0] |
3381  info.populated_ranks[channel][0][1] << 5);
3382  mchbar_write32(0x29c + (channel << 10), 0x77a);
3383  }
3384  mchbar_clrsetbits32(0x2c0, ~0, 0x6009cc00); // !!!!
3385 
3386  {
3387  u8 a, b;
3388  a = mchbar_read8(0x243);
3389  b = mchbar_read8(0x643);
3390  mchbar_write8(0x243, a | 2);
3391  mchbar_write8(0x643, b | 2);
3392  }
3393 
3394  write_1d0(7, 0x19b, 3, 1);
3395  write_1d0(7, 0x1c0, 3, 1);
3396  write_1d0(4, 0x1c6, 4, 1);
3397  write_1d0(4, 0x1cc, 4, 1);
3398  rmw_1d0(0x151, 0xf, 0x4, 4);
3399  mchbar_write32(0x584, 0xfffff);
3400  mchbar_write32(0x984, 0xfffff);
3401 
3402  for (channel = 0; channel < NUM_CHANNELS; channel++)
3403  for (slot = 0; slot < NUM_SLOTS; slot++)
3404  for (rank = 0; rank < NUM_RANKS; rank++)
3405  if (info.
3406  populated_ranks[channel][slot]
3407  [rank])
3408  config_rank(&info, s3resume,
3409  channel, slot,
3410  rank);
3411 
3412  mchbar_write8(0x243, 1);
3413  mchbar_write8(0x643, 1);
3414  }
3415 
3416  /* was == 1 but is common */
3417  pci_write_config16(NORTHBRIDGE, 0xc8, 3);
3418  write_26c(0, 0x820);
3419  write_26c(1, 0x820);
3420  mchbar_setbits32(0x130, 1 << 1);
3421  /* end */
3422 
3423  if (s3resume) {
3424  for (channel = 0; channel < NUM_CHANNELS; channel++) {
3425  mchbar_write32(0x294 + (channel << 10),
3426  (info.populated_ranks_mask[channel] & 3) << 16);
3427  mchbar_write16(0x298 + (channel << 10),
3428  info.populated_ranks[channel][0][0] |
3429  info.populated_ranks[channel][0][1] << 5);
3430  mchbar_write32(0x29c + (channel << 10), 0x77a);
3431  }
3432  mchbar_clrsetbits32(0x2c0, ~0, 0x6009cc00); // !!!!
3433  }
3434 
3435  mchbar_clrbits32(0xfa4, 1 << 24 | 1 << 1);
3436  mchbar_write32(0xfb0, 0x2000e019);
3437 
3438  /* Before training. */
3439  timestamp_add_now(103);
3440 
3441  if (!s3resume)
3442  ram_training(&info);
3443 
3444  /* After training. */
3445  timestamp_add_now(104);
3446 
3447  dump_timings(&info);
3448 
3449  program_modules_memory_map(&info, 0);
3450  program_total_memory_map(&info);
3451 
3452  if (info.non_interleaved_part_mb != 0 && info.interleaved_part_mb != 0)
3453  mchbar_write8(0x111, 0 << 2 | 1 << 5 | 1 << 6 | 0 << 7);
3454  else if (have_match_ranks(&info, 0, 4) && have_match_ranks(&info, 1, 4))
3455  mchbar_write8(0x111, 3 << 2 | 1 << 5 | 0 << 6 | 1 << 7);
3456  else if (have_match_ranks(&info, 0, 2) && have_match_ranks(&info, 1, 2))
3457  mchbar_write8(0x111, 3 << 2 | 1 << 5 | 0 << 6 | 0 << 7);
3458  else
3459  mchbar_write8(0x111, 3 << 2 | 1 << 5 | 1 << 6 | 0 << 7);
3460 
3461  mchbar_clrbits32(0xfac, 1 << 31);
3462  mchbar_write32(0xfb4, 0x4800);
3463  mchbar_write32(0xfb8, (info.revision < 8) ? 0x20 : 0x0);
3464  mchbar_write32(0xe94, 0x7ffff);
3465  mchbar_write32(0xfc0, 0x80002040);
3466  mchbar_write32(0xfc4, 0x701246);
3467  mchbar_clrbits8(0xfc8, 0x70);
3468  mchbar_setbits32(0xe5c, 1 << 24);
3469  mchbar_clrsetbits32(0x1a70, 3 << 20, 2 << 20);
3470  mchbar_write32(0x50, 0x700b0);
3471  mchbar_write32(0x3c, 0x10);
3472  mchbar_clrsetbits8(0x1aa8, 0x3f, 0xa);
3473  mchbar_setbits8(0xff4, 1 << 1);
3474  mchbar_clrsetbits32(0xff8, 0xe008, 0x1020);
3475 
3476  mchbar_write32(0xd00, IOMMU_BASE2 | 1);
3477  mchbar_write32(0xd40, IOMMU_BASE1 | 1);
3478  mchbar_write32(0xdc0, IOMMU_BASE4 | 1);
3479 
3480  write32p(IOMMU_BASE1 | 0xffc, 0x80000000);
3481  write32p(IOMMU_BASE2 | 0xffc, 0xc0000000);
3482  write32p(IOMMU_BASE4 | 0xffc, 0x80000000);
3483 
3484  {
3485  u32 eax;
3486 
3487  eax = info.fsb_frequency / 9;
3488  mchbar_clrsetbits32(0xfcc, 0x3ffff,
3489  (eax * 0x280) | (eax * 0x5000) | eax | 0x40000);
3490  mchbar_write32(0x20, 0x33001);
3491  }
3492 
3493  for (channel = 0; channel < NUM_CHANNELS; channel++) {
3494  mchbar_clrbits32(0x220 + (channel << 10), 0x7770);
3495  if (info.max_slots_used_in_channel == 1)
3496  mchbar_setbits16(0x237 + (channel << 10), 0x0201);
3497  else
3498  mchbar_clrbits16(0x237 + (channel << 10), 0x0201);
3499 
3500  mchbar_setbits8(0x241 + (channel << 10), 1 << 0);
3501 
3502  if (info.clock_speed_index <= 1 && (info.silicon_revision == 2
3503  || info.silicon_revision == 3))
3504  mchbar_setbits32(0x248 + (channel << 10), 0x00102000);
3505  else
3506  mchbar_clrbits32(0x248 + (channel << 10), 0x00102000);
3507  }
3508 
3509  mchbar_setbits32(0x115, 1 << 24);
3510 
3511  {
3512  u8 al;
3513  al = 0xd;
3514  if (!(info.silicon_revision == 0 || info.silicon_revision == 1))
3515  al += 2;
3516  al |= ((1 << (info.max_slots_used_in_channel - 1)) - 1) << 4;
3517  mchbar_write32(0x210, al << 16 | 0x20);
3518  }
3519 
3520  for (channel = 0; channel < NUM_CHANNELS; channel++) {
3521  mchbar_write32(0x288 + (channel << 10), 0x70605040);
3522  mchbar_write32(0x28c + (channel << 10), 0xfffec080);
3523  mchbar_write32(0x290 + (channel << 10), 0x282091c |
3524  (info.max_slots_used_in_channel - 1) << 0x16);
3525  }
3526  u32 reg1c;
3527  pci_read_config32(NORTHBRIDGE, 0x40); // = DEFAULT_EPBAR | 0x001 // OK
3528  reg1c = epbar_read32(EPVC1RCAP); // = 0x8001 // OK
3529  pci_read_config32(NORTHBRIDGE, 0x40); // = DEFAULT_EPBAR | 0x001 // OK
3530  epbar_write32(EPVC1RCAP, reg1c); // OK
3531  mchbar_read8(0xe08); // = 0x0
3532  pci_read_config32(NORTHBRIDGE, 0xe4); // = 0x316126
3533  mchbar_setbits8(0x1210, 1 << 1);
3534  mchbar_write32(0x1200, 0x8800440);
3535  mchbar_write32(0x1204, 0x53ff0453);
3536  mchbar_write32(0x1208, 0x19002043);
3537  mchbar_write16(0x1214, 0x320);
3538 
3539  if (info.revision == 0x10 || info.revision == 0x11) {
3540  mchbar_write16(0x1214, 0x220);
3541  mchbar_setbits8(0x1210, 1 << 6);
3542  }
3543 
3544  mchbar_setbits8(0x1214, 1 << 2);
3545  mchbar_write8(0x120c, 1);
3546  mchbar_write8(0x1218, 3);
3547  mchbar_write8(0x121a, 3);
3548  mchbar_write8(0x121c, 3);
3549  mchbar_write16(0xc14, 0);
3550  mchbar_write16(0xc20, 0);
3551  mchbar_write32(0x1c, 0);
3552 
3553  /* revision dependent here. */
3554 
3555  mchbar_setbits16(0x1230, 0x1f07);
3556 
3557  if (info.uma_enabled)
3558  mchbar_setbits32(0x11f4, 1 << 28);
3559 
3560  mchbar_setbits16(0x1230, 1 << 15);
3561  mchbar_setbits8(0x1214, 1 << 0);
3562 
3563  u8 bl, ebpb;
3564  u16 reg_1020;
3565 
3566  reg_1020 = mchbar_read32(0x1020); // = 0x6c733c // OK
3567  mchbar_write8(0x1070, 1);
3568 
3569  mchbar_write32(0x1000, 0x100);
3570  mchbar_write8(0x1007, 0);
3571 
3572  if (reg_1020 != 0) {
3573  mchbar_write16(0x1018, 0);
3574  bl = reg_1020 >> 8;
3575  ebpb = reg_1020 & 0xff;
3576  } else {
3577  ebpb = 0;
3578  bl = 8;
3579  }
3580 
3581  rdmsr(0x1a2);
3582 
3583  mchbar_write32(0x1014, 0xffffffff);
3584 
3585  mchbar_write32(0x1010, ((((ebpb + 0x7d) << 7) / bl) & 0xff) * !!reg_1020);
3586 
3587  mchbar_write8(0x101c, 0xb8);
3588 
3589  mchbar_clrsetbits8(0x123e, 0xf0, 0x60);
3590  if (reg_1020 != 0) {
3591  mchbar_clrsetbits32(0x123c, 0xf << 20, 0x6 << 20);
3592  mchbar_write8(0x101c, 0xb8);
3593  }
3594 
3595  const u64 heci_uma_addr =
3596  ((u64)
3597  ((((u64)pci_read_config16(NORTHBRIDGE, TOM)) << 6) -
3598  info.memory_reserved_for_heci_mb)) << 20;
3599 
3600  setup_heci_uma(heci_uma_addr, info.memory_reserved_for_heci_mb);
3601 
3602  if (info.uma_enabled) {
3603  u16 ax;
3604  mchbar_setbits32(0x11b0, 1 << 14);
3605  mchbar_setbits32(0x11b4, 1 << 14);
3606  mchbar_setbits16(0x1190, 1 << 14);
3607 
3608  ax = mchbar_read16(0x1190) & 0xf00; // = 0x480a // OK
3609  mchbar_write16(0x1170, ax | (mchbar_read16(0x1170) & 0x107f) | 0x4080);
3610  mchbar_setbits16(0x1170, 1 << 12);
3611 
3612  udelay(1000);
3613 
3614  u16 ecx;
3615  for (ecx = 0xffff; ecx && (mchbar_read16(0x1170) & (1 << 12)); ecx--)
3616  ;
3617  mchbar_clrbits16(0x1190, 1 << 14);
3618  }
3619 
3620  pci_write_config8(SOUTHBRIDGE, GEN_PMCON_2,
3621  pci_read_config8(SOUTHBRIDGE, GEN_PMCON_2) & ~0x80);
3622  udelay(10000);
3623  mchbar_write16(0x2ca8, 1 << 3);
3624 
3625  udelay(1000);
3626  dump_timings(&info);
3627  cbmem_wasnot_inited = cbmem_recovery(s3resume);
3628 
3629  if (!s3resume)
3630  save_timings(&info);
3631  if (s3resume && cbmem_wasnot_inited) {
3632  printk(BIOS_ERR, "Failed S3 resume.\n");
3633  ram_check_nodie(1 * MiB);
3634 
3635  /* Failed S3 resume, reset to come up cleanly */
3636  full_reset();
3637  }
3638 }
PM1_CNT
#define PM1_CNT
Definition: pm.h:27
value
pte_t value
Definition: mmu.c:91
write32
static void write32(void *addr, uint32_t val)
Definition: mmio.h:40
read8
static uint8_t read8(const void *addr)
Definition: mmio.h:12
memcpy
void * memcpy(void *dest, const void *src, size_t n)
Definition: memcpy.c:7
memset
void * memset(void *dstpp, int c, size_t len)
Definition: memset.c:12
intel_early_me_init
int intel_early_me_init(void)
Definition: early_me.c:43
intel_early_me_uma_size
int intel_early_me_uma_size(void)
Definition: early_me.c:74
ARRAY_SIZE
#define ARRAY_SIZE(a)
Definition: helpers.h:12
MIN
#define MIN(a, b)
Definition: helpers.h:37
ALIGN_DOWN
#define ALIGN_DOWN(x, a)
Definition: helpers.h:18
MiB
#define MiB
Definition: helpers.h:76
MAX
#define MAX(a, b)
Definition: helpers.h:40
ALIGN_UP
#define ALIGN_UP(x, a)
Definition: helpers.h:17
DIV_ROUND_UP
#define DIV_ROUND_UP(x, y)
Definition: helpers.h:60
cbmem_recovery
int cbmem_recovery(int s3resume)
Definition: imd_cbmem.c:125
full_reset
void full_reset(void)
Definition: cf9_reset.c:45
system_reset
void system_reset(void)
Definition: cf9_reset.c:37
cf9_reset.h
addr
static u32 addr
Definition: cirrus.c:14
printk
#define printk(level,...)
Definition: stdlib.h:16
die
void __noreturn die(const char *fmt,...)
Definition: die.c:17
RAM_DEBUG
#define RAM_DEBUG
Definition: console.h:13
RAM_SPEW
#define RAM_SPEW
Definition: console.h:14
inb
u8 inb(u16 port)
inl
u32 inl(u16 port)
outl
void outl(u32 val, u16 port)
raminit
void raminit(struct romstage_params *params)
Definition: raminit.c:15
info
static struct smmstore_params_info info
Definition: ramstage.c:12
REMAPLIMIT
#define REMAPLIMIT
Definition: e7505.h:36
REMAPBASE
#define REMAPBASE
Definition: e7505.h:35
mchbar_read8
static __always_inline uint8_t mchbar_read8(const uintptr_t offset)
Definition: fixed_bars.h:11
mchbar_setbits32
#define mchbar_setbits32(addr, set)
Definition: fixed_bars.h:58
dmibar_write8
static __always_inline void dmibar_write8(const uintptr_t offset, const uint8_t value)
Definition: fixed_bars.h:81
mchbar_write16
static __always_inline void mchbar_write16(const uintptr_t offset, const uint16_t value)
Definition: fixed_bars.h:31
mchbar_write8
static __always_inline void mchbar_write8(const uintptr_t offset, const uint8_t value)
Definition: fixed_bars.h:26
epbar_read32
static __always_inline uint32_t epbar_read32(const uintptr_t offset)
Definition: fixed_bars.h:131
mchbar_clrsetbits8
static __always_inline void mchbar_clrsetbits8(uintptr_t offset, uint8_t clear, uint8_t set)
Definition: fixed_bars.h:41
mchbar_write32
static __always_inline void mchbar_write32(const uintptr_t offset, const uint32_t value)
Definition: fixed_bars.h:36
mchbar_clrbits8
#define mchbar_clrbits8(addr, clear)
Definition: fixed_bars.h:60
mchbar_clrbits16
#define mchbar_clrbits16(addr, clear)
Definition: fixed_bars.h:61
mchbar_setbits8
#define mchbar_setbits8(addr, set)
Definition: fixed_bars.h:56
dmibar_setbits32
#define dmibar_setbits32(addr, set)
Definition: fixed_bars.h:113
mchbar_clrsetbits32
static __always_inline void mchbar_clrsetbits32(uintptr_t offset, uint32_t clear, uint32_t set)
Definition: fixed_bars.h:51
mchbar_read32
static __always_inline uint32_t mchbar_read32(const uintptr_t offset)
Definition: fixed_bars.h:21
mchbar_clrbits32
#define mchbar_clrbits32(addr, clear)
Definition: fixed_bars.h:62
mchbar_clrsetbits16
static __always_inline void mchbar_clrsetbits16(uintptr_t offset, uint16_t clear, uint16_t set)
Definition: fixed_bars.h:46
mchbar_read16
static __always_inline uint16_t mchbar_read16(const uintptr_t offset)
Definition: fixed_bars.h:16
dmibar_read8
static __always_inline uint8_t dmibar_read8(const uintptr_t offset)
Definition: fixed_bars.h:66
mchbar_setbits16
#define mchbar_setbits16(addr, set)
Definition: fixed_bars.h:57
dmibar_write16
static __always_inline void dmibar_write16(const uintptr_t offset, const uint16_t value)
Definition: fixed_bars.h:86
epbar_write32
static __always_inline void epbar_write32(const uintptr_t offset, const uint32_t value)
Definition: fixed_bars.h:146
offset
static size_t offset
Definition: flashconsole.c:16
EPVC1RCAP
#define EPVC1RCAP
Definition: gm45.h:378
halt
void __noreturn halt(void)
halt the system reliably
Definition: halt.c:6
GGC
#define GGC
Definition: host_bridge.h:9
TOLUD
#define TOLUD
Definition: host_bridge.h:61
TOUUD
#define TOUUD
Definition: host_bridge.h:57
TSEG
#define TSEG
Definition: host_bridge.h:60
DEVEN
#define DEVEN
Definition: host_bridge.h:16
TOM
#define TOM
Definition: host_bridge.h:56
NUM_SLOTS
#define NUM_SLOTS
Definition: mchbar.h:8
NUM_CHANNELS
#define NUM_CHANNELS
Definition: mchbar.h:7
clflush
static void clflush(void *addr)
Definition: cache.h:25
rdmsr
static __always_inline msr_t rdmsr(unsigned int index)
Definition: msr.h:146
wrmsr
static __always_inline void wrmsr(unsigned int index, msr_t msr)
Definition: msr.h:157
read32p
static __always_inline uint32_t read32p(const uintptr_t addr)
Definition: mmio.h:220
write32p
static __always_inline void write32p(const uintptr_t addr, const uint32_t value)
Definition: mmio.h:240
pci_ops.h
pci_write_config32
static __always_inline void pci_write_config32(const struct device *dev, u16 reg, u32 val)
Definition: pci_ops.h:76
pci_read_config16
static __always_inline u16 pci_read_config16(const struct device *dev, u16 reg)
Definition: pci_ops.h:52
pci_read_config32
static __always_inline u32 pci_read_config32(const struct device *dev, u16 reg)
Definition: pci_ops.h:58
pci_read_config8
static __always_inline u8 pci_read_config8(const struct device *dev, u16 reg)
Definition: pci_ops.h:46
pci_write_config16
static __always_inline void pci_write_config16(const struct device *dev, u16 reg, u16 val)
Definition: pci_ops.h:70
pci_write_config8
static __always_inline void pci_write_config8(const struct device *dev, u16 reg, u8 val)
Definition: pci_ops.h:64
smbus_read_byte
static int smbus_read_byte(struct device *const dev, u8 addr)
Definition: smbus.h:43
DEFAULT_PMBASE
#define DEFAULT_PMBASE
Definition: iomap.h:14
SLP_TYP_S3
#define SLP_TYP_S3
Definition: pmc.h:67
ip_checksum.h
halfcycle_ps
static unsigned int halfcycle_ps(struct raminfo *info)
Definition: raminit.h:84
frequency_11
static unsigned int frequency_11(struct raminfo *info)
Definition: raminit.h:90
fsbcycle_ps
static unsigned int fsbcycle_ps(struct raminfo *info)
Definition: raminit.h:78
NUM_RANKS
#define NUM_RANKS
Definition: raminit.h:10
reg178_step
const u8 reg178_step[]
Definition: raminit_tables.c:1234
u16_fffd0c70
const u16 u16_fffd0c70[2][2]
Definition: raminit_tables.c:1132
u16_fffd0c50
const u16 u16_fffd0c50[3][2][2]
Definition: raminit_tables.c:1137
u8_FFFD1891
const u8 u8_FFFD1891[2][2][4][12]
Definition: raminit_tables.c:605
reg178_min
const u8 reg178_min[]
Definition: raminit_tables.c:1232
lut16
const u8 lut16[4]
Definition: raminit_tables.c:1246
min_cas_latency_time
const u16 min_cas_latency_time[4]
Definition: raminit_tables.c:1156
u16_FFFE0EB8
const u16 u16_FFFE0EB8[2][4]
Definition: raminit_tables.c:573
min_cycletime
const u16 min_cycletime[4]
Definition: raminit_tables.c:1153
reg178_max
const u8 reg178_max[]
Definition: raminit_tables.c:1233
u16_fe0eb8
const u16 u16_fe0eb8[2][4]
Definition: raminit_tables.c:1241
u8_FFFD1218
const u8 u8_FFFD1218[4]
Definition: raminit_tables.c:1228
u16_ffd1188
const u16 u16_ffd1188[2][9][4]
Definition: raminit_tables.c:579
u8_FFFD0C78
const u8 u8_FFFD0C78[2][5][4][2][2][4]
Definition: raminit_tables.c:703
u16_ffd1178
const u16 u16_ffd1178[2][4]
Definition: raminit_tables.c:1236
u16_fffd0c68
const u16 u16_fffd0c68[3]
Definition: raminit_tables.c:1130
u8_FFFD1240
const u8 u8_FFFD1240[2][5][9][4][4]
Definition: raminit_tables.c:6
u8_FFFD17E0
const u8 u8_FFFD17E0[2][5][4][4]
Definition: raminit_tables.c:636
u8_FFFD0EF8
const u8 u8_FFFD0EF8[2][5][4][4]
Definition: raminit_tables.c:1160
IGD_BASE
#define IGD_BASE
Definition: host_bridge.h:22
GTT_BASE
#define GTT_BASE
Definition: host_bridge.h:23
CAPID0
#define CAPID0
Definition: host_bridge.h:29
SAD_DRAM_RULE
#define SAD_DRAM_RULE(x)
Definition: ironlake.h:52
QPI_NON_CORE
#define QPI_NON_CORE
Definition: ironlake.h:36
MAX_RTIDS
#define MAX_RTIDS
Definition: ironlake.h:38
QPI_SAD
#define QPI_SAD
Definition: ironlake.h:45
SAD_INTERLEAVE_LIST
#define SAD_INTERLEAVE_LIST(x)
Definition: ironlake.h:53
type
unsigned int type
Definition: edid.c:57
x
int x
Definition: edid.c:994
timestamp_add_now
void timestamp_add_now(enum timestamp_id id)
Definition: timestamp.c:141
ram_check_nodie
int ram_check_nodie(uintptr_t start)
Definition: ramtest.c:134
BIOS_INFO
#define BIOS_INFO
BIOS_INFO - Expected events.
Definition: loglevel.h:113
BIOS_DEBUG
#define BIOS_DEBUG
BIOS_DEBUG - Verbose output.
Definition: loglevel.h:128
BIOS_ERR
#define BIOS_ERR
BIOS_ERR - System in incomplete state.
Definition: loglevel.h:72
mrc_cache_stash_data
int mrc_cache_stash_data(int type, uint32_t version, const void *data, size_t size)
Returns < 0 on error, 0 on success.
Definition: mrc_cache.c:687
result
result
Definition: mrc_cache.c:35
mrc_cache_current_mmap_leak
void * mrc_cache_current_mmap_leak(int type, uint32_t version, size_t *data_size)
mrc_cache_mmap_leak
Definition: mrc_cache.c:342
mrc_cache.h
MRC_TRAINING_DATA
@ MRC_TRAINING_DATA
Definition: mrc_cache.h:11
IOMMU_BASE2
#define IOMMU_BASE2
Definition: memmap.h:7
IOMMU_BASE4
#define IOMMU_BASE4
Definition: memmap.h:9
IOMMU_BASE1
#define IOMMU_BASE1
Definition: memmap.h:6
MSAC
#define MSAC
Definition: haswell.h:21
MRC_CACHE_VERSION
#define MRC_CACHE_VERSION
Definition: raminit.c:31
memory_map
static struct map_entry memory_map[NUM_MAP_ENTRIES]
Definition: northbridge.c:206
get_580
static u32 get_580(int channel, u8 addr)
Definition: raminit.c:261
get_etalon2
static u32 get_etalon2(int flip, u32 addr)
Definition: raminit.c:1557
dmi_setup
static void dmi_setup(void)
Definition: raminit.c:2846
calculate_timings
static void calculate_timings(struct raminfo *info)
Definition: raminit.c:460
set_ecc
static void set_ecc(int onoff)
Definition: raminit.c:2176
get_cached_training
static const struct ram_training * get_cached_training(void)
Definition: raminit.c:1494
write_training_data
static void write_training_data(struct raminfo *info)
Definition: raminit.c:1405
get_seed2
static u32 get_seed2(int a, int b)
Definition: raminit.c:1680
set_10b
static void set_10b(struct raminfo *info, u8 val)
Definition: raminit.c:2147
dump_timings
static void dump_timings(struct raminfo *info)
Definition: raminit.c:1429
seq9
static void seq9(struct raminfo *info, int channel, int slot, int rank)
Definition: raminit.c:276
count_ranks_in_channel
static int count_ranks_in_channel(struct raminfo *info, int channel)
Definition: raminit.c:326
rw_test
static int rw_test(int rank)
Definition: raminit.c:184
enable_cache_region
static void enable_cache_region(unsigned int base, unsigned int size)
Definition: raminit.c:1588
config_rank
static void config_rank(struct raminfo *info, int s3resume, int channel, int slot, int rank)
Definition: raminit.c:337
check_bounded
static int check_bounded(unsigned short *vals, u16 bound)
Definition: raminit.c:1777
compute_derived_timings
static void compute_derived_timings(struct raminfo *info)
Definition: raminit.c:707
rmw_1d0
static void rmw_1d0(u16 addr, u32 and, u32 or, int split)
Definition: raminit.c:417
frequency_01
static unsigned int frequency_01(struct raminfo *info)
Definition: raminit.c:692
ps_to_halfcycles
static unsigned int ps_to_halfcycles(struct raminfo *info, unsigned int ps)
Definition: raminit.c:697
program_timings
static void program_timings(struct raminfo *info, u16 base, int channel, int slot, int rank)
Definition: raminit.c:225
train_ram_at_178
static void train_ram_at_178(struct raminfo *info, u8 channel, int slot, int rank, u8 total_rank, u8 reg_178, int first_run, int niter, timing_bounds_t *timings)
Definition: raminit.c:1853
read128
static void read128(u32 addr, u64 *out)
Definition: raminit.c:63
write_testing_type2
static void write_testing_type2(struct raminfo *info, u8 totalrank, u8 region, u8 block, char flip)
Definition: raminit.c:1735
ns_to_cycles
static unsigned int ns_to_cycles(struct raminfo *info, unsigned int ns)
Definition: raminit.c:702
NORTHBRIDGE
#define NORTHBRIDGE
Definition: raminit.c:35
DEFAULT_PCI_MMIO_SIZE
#define DEFAULT_PCI_MMIO_SIZE
Definition: raminit.c:1248
gav_real
static u32 gav_real(int line, u32 in)
Definition: raminit.c:131
clear_errors
static void clear_errors(void)
Definition: raminit.c:1610
rmw_500
static void rmw_500(struct raminfo *info, int channel, u16 addr, int bits, u32 and, u32 or)
Definition: raminit.c:178
get_bits_420
static u8 get_bits_420(const u32 reg32)
Definition: raminit.c:2910
choose_training
static u8 choose_training(struct raminfo *info, int channel, int slot, int rank, int lane, timing_bounds_t *timings, u8 center_178)
Definition: raminit.c:2285
collect_system_info
static void collect_system_info(struct raminfo *info)
Definition: raminit.c:1347
choose_reg178
static u8 choose_reg178(struct raminfo *info, timing_bounds_t *timings)
Definition: raminit.c:2368
disable_cache_region
static void disable_cache_region(void)
Definition: raminit.c:1580
MR2_RZQ2
@ MR2_RZQ2
Definition: raminit.c:948
MR2_RTT_WR_DISABLED
@ MR2_RTT_WR_DISABLED
Definition: raminit.c:947
write_testing
static void write_testing(struct raminfo *info, int totalrank, int flip)
Definition: raminit.c:1615
do_ram_training
static void do_ram_training(struct raminfo *info)
Definition: raminit.c:2618
timings_car
timing_bounds_t timings_car[64]
Definition: raminit.c:140
toggle_1d0_142_5ff
static void toggle_1d0_142_5ff(void)
Definition: raminit.c:249
check_testing
static u8 check_testing(struct raminfo *info, u8 total_rank, int flip)
Definition: raminit.c:1636
DEVICE_TYPE
@ DEVICE_TYPE
Definition: raminit.c:443
MODULE_TYPE
@ MODULE_TYPE
Definition: raminit.c:444
THERMAL_AND_REFRESH
@ THERMAL_AND_REFRESH
Definition: raminit.c:455
CAS_LATENCY_TIME
@ CAS_LATENCY_TIME
Definition: raminit.c:454
TIMEBASE_DIVISOR
@ TIMEBASE_DIVISOR
Definition: raminit.c:449
RANK1_ADDRESS_MAPPING
@ RANK1_ADDRESS_MAPPING
Definition: raminit.c:457
DENSITY
@ DENSITY
Definition: raminit.c:445
CAS_LATENCIES_LSB
@ CAS_LATENCIES_LSB
Definition: raminit.c:452
REFERENCE_RAW_CARD_USED
@ REFERENCE_RAW_CARD_USED
Definition: raminit.c:456
TIMEBASE_DIVIDEND
@ TIMEBASE_DIVIDEND
Definition: raminit.c:448
CYCLETIME
@ CYCLETIME
Definition: raminit.c:450
RANKS_AND_DQ
@ RANKS_AND_DQ
Definition: raminit.c:446
CAS_LATENCIES_MSB
@ CAS_LATENCIES_MSB
Definition: raminit.c:453
MEMORY_BUS_WIDTH
@ MEMORY_BUS_WIDTH
Definition: raminit.c:447
get_max_timing
u16 get_max_timing(struct raminfo *info, int channel)
Definition: raminit.c:2824
seed1
const u32 seed1[0x18]
Definition: raminit.c:1671
SOUTHBRIDGE
#define SOUTHBRIDGE
Definition: raminit.c:36
set_178
static void set_178(u8 val)
Definition: raminit.c:2190
save_timings
static void save_timings(struct raminfo *info)
Definition: raminit.c:1460
validate_state
static int validate_state(enum state *in)
Definition: raminit.c:1791
ram_training
static void ram_training(struct raminfo *info)
Definition: raminit.c:2806
find_highest_bit_set
static int find_highest_bit_set(u16 val)
Definition: raminit.c:424
write_26c
static void write_26c(int channel, u16 si)
Definition: raminit.c:242
MR1_ODS34OHM
@ MR1_ODS34OHM
Definition: raminit.c:938
MR1_RZQ4
@ MR1_RZQ4
Definition: raminit.c:937
MR1_RZQ2
@ MR1_RZQ2
Definition: raminit.c:936
MR1_RZQ12
@ MR1_RZQ12
Definition: raminit.c:935
GMA
#define GMA
Definition: raminit.c:37
make_shift
static int make_shift(int comp2, int comp5, int x)
Definition: raminit.c:1691
get_timing_register_addr
static u16 get_timing_register_addr(int lane, int tm, int slot, int rank)
Definition: raminit.c:125
write_1d0
static void write_1d0(u32 val, u16 addr, int bits, int flag)
Definition: raminit.c:85
cycle_ps
static unsigned int cycle_ps(struct raminfo *info)
Definition: raminit.c:686
program_base_timings
static void program_base_timings(struct raminfo *info)
Definition: raminit.c:538
program_total_memory_map
static void program_total_memory_map(struct raminfo *info)
Definition: raminit.c:1250
FOR_POPULATED_RANKS_BACKWARDS
#define FOR_POPULATED_RANKS_BACKWARDS
Definition: raminit.c:50
program_board_delay
static void program_board_delay(struct raminfo *info)
Definition: raminit.c:1078
get_etalon
static u32 get_etalon(int flip, u32 addr)
Definition: raminit.c:1703
read_4090
static void read_4090(struct raminfo *info)
Definition: raminit.c:1529
check_testing_type2
static u8 check_testing_type2(struct raminfo *info, u8 totalrank, u8 region, u8 block, char flip)
Definition: raminit.c:1745
do_fsm
static void do_fsm(enum state *state, u16 *counter, u8 fail_mask, int margin, int uplimit, u8 *res_low, u8 *res_high, u8 val)
Definition: raminit.c:1801
set_334
static void set_334(int zero)
Definition: raminit.c:375
write_500_timings_type
static void write_500_timings_type(struct raminfo *info, int channel, int slot, int rank, int type)
Definition: raminit.c:2201
get_lane_offset
static u16 get_lane_offset(int slot, int rank, int lane)
Definition: raminit.c:119
STANDARD_MIN_MARGIN
#define STANDARD_MIN_MARGIN
Definition: raminit.c:2366
write_500
static void write_500(struct raminfo *info, int channel, u32 val, u16 addr, int bits, int flag)
Definition: raminit.c:161
have_match_ranks
static int have_match_ranks(struct raminfo *info, int channel, int ranks)
Definition: raminit.c:1501
chipset_init
void chipset_init(const int s3resume)
Definition: raminit.c:2860
check_cached_sanity
static int check_cached_sanity(struct raminfo *info)
Definition: raminit.c:2428
flush_cache
static void flush_cache(u32 start, u32 size)
Definition: raminit.c:1600
sfence
static void sfence(void)
Definition: raminit.c:114
FOR_ALL_RANKS
#define FOR_ALL_RANKS
Definition: raminit.c:39
find_lowest_bit_set32
static int find_lowest_bit_set32(u32 val)
Definition: raminit.c:433
read_500
static u16 read_500(struct raminfo *info, int channel, u16 addr, int split)
Definition: raminit.c:144
jedec_init
static void jedec_init(struct raminfo *info)
Definition: raminit.c:951
state
state
Definition: raminit.c:1787
COMPLETE
@ COMPLETE
Definition: raminit.c:1788
AT_MARGIN
@ AT_MARGIN
Definition: raminit.c:1788
AT_USABLE
@ AT_USABLE
Definition: raminit.c:1788
BEFORE_USABLE
@ BEFORE_USABLE
Definition: raminit.c:1788
gav
#define gav(x)
Definition: raminit.c:137
program_modules_memory_map
static void program_modules_memory_map(struct raminfo *info, int pre_jedec)
Definition: raminit.c:1044
read_1d0
static u16 read_1d0(u16 addr, int split)
Definition: raminit.c:97
try_cached_training
static int try_cached_training(struct raminfo *info)
Definition: raminit.c:2463
u128
struct _u128 u128
set_4cf
static void set_4cf(struct raminfo *info, int channel, u8 bit, u8 val)
Definition: raminit.c:366
jedec_read
static void jedec_read(struct raminfo *info, int channel, int slot, int rank, int total_rank, u8 addr3, unsigned int value)
Definition: raminit.c:913
MR0_DLL_RESET_ON
@ MR0_DLL_RESET_ON
Definition: raminit.c:943
MR0_BT_INTERLEAVED
@ MR0_BT_INTERLEAVED
Definition: raminit.c:942
try_timing_offsets
static void try_timing_offsets(struct raminfo *info, int channel, int slot, int rank, int totalrank)
Definition: raminit.c:2215
FOR_POPULATED_RANKS
#define FOR_POPULATED_RANKS
Definition: raminit.c:44
get_uint_option
unsigned int get_uint_option(const char *name, const unsigned int fallback)
Definition: option.c:116
pci_def.h
PCI_DEVICE_ID
#define PCI_DEVICE_ID
Definition: pci_def.h:9
PCI_REVISION_ID
#define PCI_REVISION_ID
Definition: pci_def.h:41
read_pmbase32
u32 read_pmbase32(const u8 addr)
Definition: pmbase.c:57
write_pmbase32
void write_pmbase32(const u8 addr, const u32 val)
Definition: pmbase.c:36
late_quickpath_init
void late_quickpath_init(struct raminfo *info, const int s3resume)
Definition: quickpath.c:592
early_quickpath_init
void early_quickpath_init(struct raminfo *info, const u8 x2ca8)
Definition: quickpath.c:476
raminit_tables.h
smbus_host.h
base
uintptr_t base
Definition: uart.c:17
GEN_PMCON_2
#define GEN_PMCON_2
Definition: lpc.h:58
mask
static const int mask[4]
Definition: gpio.c:308
DEFAULT_GPIOBASE
#define DEFAULT_GPIOBASE
Definition: pch.h:22
RCBA8
#define RCBA8(x)
Definition: rcba.h:12
RCBA32
#define RCBA32(x)
Definition: rcba.h:14
setup_heci_uma
void setup_heci_uma(u64 heci_uma_addr, unsigned int heci_uma_size)
Definition: setup_heci_uma.c:201
NULL
#define NULL
Definition: stddef.h:19
u64
uint64_t u64
Definition: stdint.h:54
u32
uint32_t u32
Definition: stdint.h:51
uintptr_t
unsigned long uintptr_t
Definition: stdint.h:21
u16
uint16_t u16
Definition: stdint.h:48
u8
uint8_t u8
Definition: stdint.h:45
_u128
Definition: raminit.c:58
_u128::lo
u64 lo
Definition: raminit.c:59
_u128::hi
u64 hi
Definition: raminit.c:60
in
Definition: jpeg.c:27
msr_struct
Definition: msr.h:110
msr_struct::hi
unsigned int hi
Definition: msr.h:112
msr_struct::lo
unsigned int lo
Definition: msr.h:111
ram_training
Definition: raminit.h:20
ram_training::timing_offset
u16 timing_offset[2][2][2][9]
Definition: raminit.h:30
ram_training::reg178_largest
u8 reg178_largest
Definition: raminit.h:28
ram_training::reg_178
u16 reg_178
Definition: raminit.h:23
ram_training::reg_10b
u16 reg_10b
Definition: raminit.h:24
ram_training::timing2_offset
u16 timing2_offset[2][2][2][9]
Definition: raminit.h:31
ram_training::reg274265
u8 reg274265[2][3]
Definition: raminit.h:33
ram_training::timing_bounds
timing_bounds_t timing_bounds[2]
Definition: raminit.h:29
ram_training::reg178_center
u8 reg178_center
Definition: raminit.h:26
ram_training::reg2ca9_bit0
u8 reg2ca9_bit0
Definition: raminit.h:34
ram_training::reg178_smallest
u8 reg178_smallest
Definition: raminit.h:27
ram_training::timing2_bounds
u16 timing2_bounds[2][2][2][9][2]
Definition: raminit.h:32
ram_training::reg_6dc
u32 reg_6dc
Definition: raminit.h:35
ram_training::lane_timings
u16 lane_timings[4][2][2][2][9]
Definition: raminit.h:22
ram_training::reg_6e8
u32 reg_6e8
Definition: raminit.h:36
raminfo::spd
u8 spd[2][2][151]
Definition: raminit.h:52
raminfo::populated_ranks
u8 populated_ranks[2][2][2]
Definition: raminit.h:44
region
Definition: region.h:76
timing_bounds_t
Definition: raminit.h:13
timing_bounds_t::smallest
u8 smallest
Definition: raminit.h:14
val
u8 val
Definition: sys.c:300
c
#define c(value, pmcreg, dst_bits)
udelay
void udelay(uint32_t us)
Definition: udelay.c:15
count
#define count
MTRR_PHYS_BASE
#define MTRR_PHYS_BASE(reg)
Definition: mtrr.h:39
MTRR_PHYS_MASK
#define MTRR_PHYS_MASK(reg)
Definition: mtrr.h:40
MTRR_TYPE_WRPROT
#define MTRR_TYPE_WRPROT
Definition: mtrr.h:13
MTRR_DEF_TYPE_EN
#define MTRR_DEF_TYPE_EN
Definition: mtrr.h:27