nold/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere synced 2024-11-15 17:39:28 +00:00
Code Issues Projects Releases Packages Wiki Activity
Atmosphere/fusee/fusee-mtc/src/mtc.c

3750 lines
186 KiB
C
Raw Normal View History

fusee: Implement DRAM training: - Based on reverse engineered code and Peter De Schrijver's patches; - Complemented with CTCaer's minerva_tc project.
2018-11-12 01:55:16 +00:00
/*
* Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2018 CTCaer <ctcaer@gmail.com>
find -exec sed -i'' -e 's/2018-2019 Atmo/2018-2020 Atmo/g' {} +
2020-01-24 10:10:40 +00:00
* Copyright (c) 2018-2020 Atmosphère-NX
fusee: Implement DRAM training: - Based on reverse engineered code and Peter De Schrijver's patches; - Complemented with CTCaer's minerva_tc project.
2018-11-12 01:55:16 +00:00
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mtc.h"
#include "mtc_tables.h"
#include "car.h"
#include "fuse.h"
#include "timers.h"
#include "lib/log.h"
/*
* Macros.
*/
#define max(x, y) ({ \
typeof(x) _max1 = (x); \
typeof(y) _max2 = (y); \
(void) (&_max1 == &_max2); \
_max1 > _max2 ? _max1 : _max2; })
#define max_t(type, x, y) ({ \
type __max1 = (x); \
type __max2 = (y); \
__max1 > __max2 ? __max1: __max2; })
#define TRIM_REG(chan, rank, reg, byte) \
((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK & \
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## \
rank ## _ ## reg ## _INDEX]) >> \
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT) \
+ \
(((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
byte ## _DATA_BRLSHFT_MASK & \
next_timing->trim_perch_regs[REG_EMC ## chan ## \
_EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >> \
EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
byte ## _DATA_BRLSHFT_SHIFT) * 64)
#define CALC_TEMP(rank, reg, byte1, byte2, n) \
((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## \
reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK) \
| \
((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## \
reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK) \
/*
* PTFV defines - basically just indexes into the per table PTFV array.
*/
#define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0
#define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1
#define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2
#define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3
#define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4
#define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5
#define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6
#define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7
#define PTFV_WRITE_SAMPLES_INDEX 8
#define PTFV_DVFS_SAMPLES_INDEX 9
#define PTFV_MOVAVG_WEIGHT_INDEX 10
#define PTFV_CONFIG_CTRL_INDEX 11
#define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0)
/*
* Do arithmetic in fixed point.
*/
#define MOVAVG_PRECISION_FACTOR 100
/*
* The division portion of the average operation.
*/
#define __AVERAGE_PTFV(dev) \
({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \
next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
next_timing->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
/*
* The division portion of the average write operation.
*/
#define __AVERAGE_WRITE_PTFV(dev) \
({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \
next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
next_timing->ptfv_list[PTFV_WRITE_SAMPLES_INDEX]; })
/*
* Convert val to fixed point and add it to the temporary average.
*/
#define __INCREMENT_PTFV(dev, val) \
({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \
((val) * MOVAVG_PRECISION_FACTOR); })
/*
* Convert a moving average back to integral form and return the value.
*/
#define __MOVAVG_AC(timing, dev) \
((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
MOVAVG_PRECISION_FACTOR)
/* Weighted update. */
#define __WEIGHTED_UPDATE_PTFV(dev, nval) \
do { \
int w = PTFV_MOVAVG_WEIGHT_INDEX; \
int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \
\
next_timing->ptfv_list[dqs] = \
((nval * MOVAVG_PRECISION_FACTOR) + \
(next_timing->ptfv_list[dqs] * \
next_timing->ptfv_list[w])) / \
(next_timing->ptfv_list[w] + 1); \
} while (0)
/* Access a particular average. */
#define __MOVAVG(timing, dev) \
((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX])
static int g_active_timing_table_idx = -1;
static bool g_is_pllmb = false;
static bool g_fsp_for_next_freq = false;
static bool g_write_training_pattern = true;
#define DEFINE_REG(type, reg) (reg)
static const uint32_t burst_regs_per_ch_off[] = BURST_REGS_PER_CH_LIST;
static const uint32_t burst_regs_off[] = BURST_REGS_LIST;
static const uint32_t trim_regs_per_ch_off[] = TRIM_REGS_PER_CH_LIST;
static const uint32_t trim_regs_off[] = TRIM_REGS_LIST;
static const uint32_t vref_regs_per_ch_off[] = VREF_REGS_PER_CH_LIST;
static const uint32_t training_mod_regs_per_ch_off[] = TRAINING_MOD_REGS_PER_CH_LIST;
static const uint32_t burst_mc_regs_off[] = BURST_MC_REGS_LIST;
static const uint32_t la_scale_regs_off[] = BURST_UP_DOWN_REGS_LIST;
#undef DEFINE_REG
#define DEFINE_REG(type, reg) (type)
static const uint32_t burst_regs_per_ch_type[] = BURST_REGS_PER_CH_LIST;
static const uint32_t trim_regs_per_ch_type[] = TRIM_REGS_PER_CH_LIST;
static const uint32_t vref_regs_per_ch_type[] = VREF_REGS_PER_CH_LIST;
static const uint32_t training_mod_regs_per_ch_type[] = TRAINING_MOD_REGS_PER_CH_LIST;
#undef DEFINE_REG
static const uint32_t g_ram_pattern_dq[0x500] = {
0x18181818, 0x61616161, 0x85858585, 0x14141414, 0x51515151,
0x47474747, 0x1E1E1E1E, 0x79797979, 0xE5E5E5E5, 0x94949494,
0x51515151, 0x46464646, 0x19191919, 0x67676767, 0x9C9C9C9C,
0x71717171, 0xC5C5C5C5, 0x17171717, 0x5F5F5F5F, 0x7E7E7E7E,
0xFBFBFBFB, 0xEDEDEDED, 0xB4B4B4B4, 0xD2D2D2D2, 0x48484848,
0x21212121, 0x85858585, 0x16161616, 0x59595959, 0x66666666,
0x9A9A9A9A, 0x69696969, 0xA4A4A4A4, 0x93939393, 0x4F4F4F4F,
0x3F3F3F3F, 0xFCFCFCFC, 0xF3F3F3F3, 0xCDCDCDCD, 0x37373737,
0xDCDCDCDC, 0x70707070, 0xC3C3C3C3, 0x0F0F0F0F, 0x3E3E3E3E,
0xFAFAFAFA, 0xEBEBEBEB, 0xACACACAC, 0xB3B3B3B3, 0xCCCCCCCC,
0x31313131, 0xC5C5C5C5, 0x15151515, 0x57575757, 0x5F5F5F5F,
0x7F7F7F7F, 0xFDFDFDFD, 0xF4F4F4F4, 0xD0D0D0D0, 0x42424242,
0x08080808, 0x23232323, 0x8F8F8F8F, 0x3F3F3F3F, 0x18181818,
0x61616161, 0x85858585, 0x14141414, 0x51515151, 0x47474747,
0x1E1E1E1E, 0x79797979, 0xE5E5E5E5, 0x94949494, 0x51515151,
0x46464646, 0x19191919, 0x67676767, 0x9C9C9C9C, 0x71717171,
0xC5C5C5C5, 0x17171717, 0x5F5F5F5F, 0x7E7E7E7E, 0xFBFBFBFB,
0xEDEDEDED, 0xB4B4B4B4, 0xD2D2D2D2, 0x48484848, 0x21212121,
0x85858585, 0x16161616, 0x59595959, 0x66666666, 0x9A9A9A9A,
0x69696969, 0xA4A4A4A4, 0x93939393, 0x4F4F4F4F, 0x3F3F3F3F,
0xFCFCFCFC, 0xF3F3F3F3, 0xCDCDCDCD, 0x37373737, 0xDCDCDCDC,
0x70707070, 0xC3C3C3C3, 0x0F0F0F0F, 0x3E3E3E3E, 0xFAFAFAFA,
0xEBEBEBEB, 0xACACACAC, 0xB3B3B3B3, 0xCCCCCCCC, 0x31313131,
0xC5C5C5C5, 0x15151515, 0x57575757, 0x5F5F5F5F, 0x7F7F7F7F,
0xFDFDFDFD, 0xF4F4F4F4, 0xD0D0D0D0, 0x42424242, 0x08080808,
0x23232323, 0x8F8F8F8F, 0x3F3F3F3F, 0x06060606, 0x18181818,
0x21212121, 0x05050505, 0x14141414, 0x11111111, 0x07070707,
0x1E1E1E1E, 0x39393939, 0x25252525, 0x14141414, 0x11111111,
0x06060606, 0x19191919, 0x27272727, 0x1C1C1C1C, 0x31313131,
0x05050505, 0x17171717, 0x1F1F1F1F, 0x3E3E3E3E, 0x3B3B3B3B,
0x2D2D2D2D, 0x34343434, 0x12121212, 0x08080808, 0x21212121,
0x05050505, 0x16161616, 0x19191919, 0x26262626, 0x1A1A1A1A,
0x29292929, 0x24242424, 0x13131313, 0x0F0F0F0F, 0x3F3F3F3F,
0x3C3C3C3C, 0x33333333, 0x0D0D0D0D, 0x37373737, 0x1C1C1C1C,
0x30303030, 0x03030303, 0x0F0F0F0F, 0x3E3E3E3E, 0x3A3A3A3A,
0x2B2B2B2B, 0x2C2C2C2C, 0x33333333, 0x0C0C0C0C, 0x31313131,
0x05050505, 0x15151515, 0x17171717, 0x1F1F1F1F, 0x3F3F3F3F,
0x3D3D3D3D, 0x34343434, 0x10101010, 0x02020202, 0x08080808,
0x23232323, 0x0F0F0F0F, 0x06060606, 0x18181818, 0x21212121,
0x05050505, 0x14141414, 0x11111111, 0x07070707, 0x1E1E1E1E,
0x39393939, 0x25252525, 0x14141414, 0x11111111, 0x06060606,
0x19191919, 0x27272727, 0x1C1C1C1C, 0x31313131, 0x05050505,
0x17171717, 0x1F1F1F1F, 0x3E3E3E3E, 0x3B3B3B3B, 0x2D2D2D2D,
0x34343434, 0x12121212, 0x08080808, 0x21212121, 0x05050505,
0x16161616, 0x19191919, 0x26262626, 0x1A1A1A1A, 0x29292929,
0x24242424, 0x13131313, 0x0F0F0F0F, 0x3F3F3F3F, 0x3C3C3C3C,
0x33333333, 0x0D0D0D0D, 0x37373737, 0x1C1C1C1C, 0x30303030,
0x03030303, 0x0F0F0F0F, 0x3E3E3E3E, 0x3A3A3A3A, 0x2B2B2B2B,
0x2C2C2C2C, 0x33333333, 0x0C0C0C0C, 0x31313131, 0x05050505,
0x15151515, 0x17171717, 0x1F1F1F1F, 0x3F3F3F3F, 0x3D3D3D3D,
0x34343434, 0x10101010, 0x02020202, 0x08080808, 0x23232323,
0x0F0F0F0F,
0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000,
0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x00000000,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F,
0x80808080, 0x00000000, 0x80808080, 0x00000000, 0x80808080,
0x00000000, 0x80808080, 0x40404040, 0x00000000, 0x40404040,
0x00000000, 0x40404040, 0x00000000, 0x40404040, 0x20202020,
0x00000000, 0x20202020, 0x00000000, 0x20202020, 0x00000000,
0x20202020, 0x10101010, 0x00000000, 0x10101010, 0x00000000,
0x10101010, 0x00000000, 0x10101010, 0x08080808, 0x00000000,
0x08080808, 0x00000000, 0x08080808, 0x00000000, 0x08080808,
0x04040404, 0x00000000, 0x04040404, 0x00000000, 0x04040404,
0x00000000, 0x04040404, 0x02020202, 0x00000000, 0x02020202,
0x00000000, 0x02020202, 0x00000000, 0x02020202, 0x01010101,
0x00000000, 0x01010101, 0x00000000, 0x01010101, 0x00000000,
0x01010101, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x80808080,
0x00000000, 0x80808080, 0x00000000, 0x80808080, 0x00000000,
0x80808080, 0x40404040, 0x00000000, 0x40404040, 0x00000000,
0x40404040, 0x00000000, 0x40404040, 0x20202020, 0x00000000,
0x20202020, 0x00000000, 0x20202020, 0x00000000, 0x20202020,
0x10101010, 0x00000000, 0x10101010, 0x00000000, 0x10101010,
0x00000000, 0x10101010, 0x08080808, 0x00000000, 0x08080808,
0x00000000, 0x08080808, 0x00000000, 0x08080808, 0x04040404,
0x00000000, 0x04040404, 0x00000000, 0x04040404, 0x00000000,
0x04040404, 0x02020202, 0x00000000, 0x02020202, 0x00000000,
0x02020202, 0x00000000, 0x02020202, 0x01010101, 0x00000000,
0x01010101, 0x00000000, 0x01010101, 0x00000000, 0x01010101,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x20202020,
0x00000000, 0x20202020, 0x00000000, 0x20202020, 0x00000000,
0x20202020, 0x00000000, 0x20202020, 0x00000000, 0x10101010,
0x00000000, 0x10101010, 0x00000000, 0x10101010, 0x00000000,
0x10101010, 0x00000000, 0x10101010, 0x00000000, 0x08080808,
0x00000000, 0x08080808, 0x00000000, 0x08080808, 0x00000000,
0x08080808, 0x00000000, 0x08080808, 0x00000000, 0x04040404,
0x00000000, 0x04040404, 0x00000000, 0x04040404, 0x00000000,
0x04040404, 0x00000000, 0x04040404, 0x00000000, 0x02020202,
0x00000000, 0x02020202, 0x00000000, 0x02020202, 0x00000000,
0x02020202, 0x00000000, 0x02020202, 0x00000000, 0x01010101,
0x00000000, 0x01010101, 0x00000000, 0x01010101, 0x00000000,
0x01010101, 0x00000000, 0x01010101, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x20202020, 0x00000000,
0x20202020, 0x00000000, 0x20202020, 0x00000000, 0x20202020,
0x00000000, 0x20202020, 0x00000000, 0x10101010, 0x00000000,
0x10101010, 0x00000000, 0x10101010, 0x00000000, 0x10101010,
0x00000000, 0x10101010, 0x00000000, 0x08080808, 0x00000000,
0x08080808, 0x00000000, 0x08080808, 0x00000000, 0x08080808,
0x00000000, 0x08080808, 0x00000000, 0x04040404, 0x00000000,
0x04040404, 0x00000000, 0x04040404, 0x00000000, 0x04040404,
0x00000000, 0x04040404, 0x00000000, 0x02020202, 0x00000000,
0x02020202, 0x00000000, 0x02020202, 0x00000000, 0x02020202,
0x00000000, 0x02020202, 0x00000000, 0x01010101, 0x00000000,
0x01010101, 0x00000000, 0x01010101, 0x00000000, 0x01010101,
0x00000000, 0x01010101, 0x00000000, 0x00000000, 0x00000000,
0x00000000,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333
};
static const uint32_t g_ram_pattern_dmi[0x500] = {
0xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,
0x0, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,
0xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0x0, 0x0,
0xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0x0,
0xF, 0xF, 0xF, 0x0, 0xF, 0xF, 0xF, 0x0,
0x0, 0xF, 0xF, 0x0, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF,
0x0, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,
0xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,
0x0, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,
0xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0x0, 0x0,
0xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0x0,
0xF, 0xF, 0xF, 0x0, 0xF, 0xF, 0xF, 0x0,
0x0, 0xF, 0xF, 0x0, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF,
0x0, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0, 0x0,
0xF, 0xF, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0,
0xF, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0xF,
0x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,
0xF, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,
0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF,
0xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0,
0xF, 0xF, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,
0x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0, 0x0,
0xF, 0xF, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0,
0xF, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0xF,
0x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,
0xF, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,
0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF,
0xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0,
0xF, 0xF, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3
};
/*
* Register read/write helpers.
*/
static inline void emc_write(uint32_t val, uint32_t offset) {
MAKE_EMC_REG(offset) = val;
}
static inline uint32_t emc_read(uint32_t offset) {
return MAKE_EMC_REG(offset);
}
static inline void emc0_write(uint32_t val, uint32_t offset) {
MAKE_EMC0_REG(offset) = val;
}
static inline uint32_t emc0_read(uint32_t offset) {
return MAKE_EMC0_REG(offset);
}
static inline void emc1_write(uint32_t val, uint32_t offset) {
MAKE_EMC1_REG(offset) = val;
}
static inline uint32_t emc1_read(uint32_t offset) {
return MAKE_EMC1_REG(offset);
}
static inline void emc_write_per_ch(uint32_t val, int type, uint32_t offset) {
switch (type) {
case REG_EMC:
emc_write(val, offset);
break;
case REG_EMC0:
emc0_write(val, offset);
break;
case REG_EMC1:
emc1_write(val, offset);
break;
}
}
static inline uint32_t emc_read_per_ch(int type, uint32_t offset) {
uint32_t val = 0;
switch (type) {
case REG_EMC:
val = emc_read(offset);
break;
case REG_EMC0:
val = emc0_read(offset);
break;
case REG_EMC1:
val = emc1_read(offset);
break;
}
return val;
}
static inline void mc_write(uint32_t val, uint32_t offset) {
MAKE_MC_REG(offset) = val;
}
static inline uint32_t mc_read(uint32_t offset) {
return MAKE_MC_REG(offset);
}
/* Configure clock change sequence FIFO */
static void ccfifo_write(uint32_t ccfifo_addr, uint32_t ccfifo_data, uint32_t ccfifo_stall_cnt) {
MAKE_EMC_REG(EMC_CCFIFO_DATA) = ccfifo_data;
MAKE_EMC_REG(EMC_CCFIFO_ADDR) = ((ccfifo_addr & 0xFFFF) | ((ccfifo_stall_cnt & 0x7FFF) << 16) | 0x80000000);
}
static void start_periodic_compensation() {
uint32_t mpc_req = 0x4B;
// Write to EMC_MPC_0
emc_write(mpc_req, EMC_MPC);
// Dummy read
mpc_req = emc_read(EMC_MPC);
}
static uint32_t actual_osc_clocks(uint32_t in) {
if (in < 0x40)
return in * 16;
else if (in < 0x80)
return 2048;
else if (in < 0xc0)
return 4096;
else
return 8192;
}
static void emc_set_shadow_bypass(int set) {
uint32_t emc_dbg = emc_read(EMC_DBG);
if (set)
emc_write(emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
else
emc_write(emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
}
static uint32_t wait_for_update(uint32_t status_reg, uint32_t bit_mask, bool updated_state, int chan) {
for (int i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) {
if (chan == REG_EMC) {
if (((emc_read_per_ch(REG_EMC, status_reg) & bit_mask) != 0) == updated_state)
return 0;
} else {
if (((emc_read_per_ch(REG_EMC1, status_reg) & bit_mask) != 0) == updated_state)
return 0;
}
udelay(1);
}
// Timeout
return 4;
}
static void emc_timing_update(bool dual_chan) {
/* Trigger the timing update event. */
emc_write(0x1, EMC_TIMING_CONTROL);
/* Wait for the update to finish. */
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_TIMING_UPDATE_STALLED, false, REG_EMC);
if (dual_chan)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_TIMING_UPDATE_STALLED, false, REG_EMC1);
}
static uint32_t get_dll_state(tegra_emc_timing_t *next_timing) {
bool next_dll_enabled = !(next_timing->emc_emrs & 0x1);
if (next_dll_enabled)
return DLL_ON;
else
return DLL_OFF;
}
static uint32_t div_o3(uint32_t a, uint32_t b) {
uint32_t result = a / b;
if ((b * result) < a)
return result + 1;
else
return result;
}
static uint32_t dvfs_power_ramp_down(bool flip_backward, tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, uint32_t clk) {
uint32_t ramp_down_wait = 0;
uint32_t seq_wait = 0;
uint32_t pmacro_cmd_pad = 0;
uint32_t pmacro_dq_pad = 0;
uint32_t pmacro_cfg5 = 0;
uint32_t pmacro_rfu1 = 0;
uint32_t pmacro_common_tx = 0;
if (flip_backward) {
pmacro_cmd_pad = next_timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->burst_regs[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else {
pmacro_cmd_pad = current_timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = ((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 0x101) | current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]);
pmacro_rfu1 = current_timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = current_timing->burst_regs[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = current_timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
}
pmacro_cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, 0);
ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS, 12);
ramp_down_wait = clk * 12;
seq_wait = (100000 / clk) + 1;
if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) {
if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
pmacro_cmd_pad &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
pmacro_cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, seq_wait);
ramp_down_wait += 100000;
pmacro_dq_pad &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
pmacro_dq_pad |= EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0);
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x01120112, 0);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x01120112, seq_wait);
ramp_down_wait += 100000;
}
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x01bf01bf, seq_wait);
ramp_down_wait += 100000;
if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
pmacro_cmd_pad &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, seq_wait);
ramp_down_wait += 100000;
pmacro_dq_pad &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0);
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x07ff07ff, 0);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x07ff07ff, seq_wait);
ramp_down_wait += 100000;
}
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x07ff07ff, seq_wait + 19);
ramp_down_wait += (100000 + (20 * clk));
}
if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) {
ramp_down_wait += 100000;
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & ~0x5, seq_wait);
ramp_down_wait += 100000;
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & ~0xf, seq_wait);
ramp_down_wait += 100000;
ccfifo_write(0, 0, seq_wait);
ramp_down_wait += 100000;
} else {
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & ~0xf, seq_wait);
}
return ramp_down_wait;
}
static uint32_t dvfs_power_ramp_up(bool flip_backward, tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, uint32_t training, uint32_t clk) {
uint32_t ramp_up_wait = 0;
uint32_t pmacro_cmd_pad = 0;
uint32_t pmacro_dq_pad = 0;
uint32_t pmacro_cfg5 = 0;
uint32_t pmacro_rfu1 = 0;
uint32_t pmacro_common_tx = 0;
if (flip_backward) {
pmacro_cmd_pad = current_timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = current_timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = current_timing->burst_regs[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = current_timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else if (training & 3) {
pmacro_cmd_pad = next_timing->shadow_regs_ca_train[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->shadow_regs_ca_train[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->shadow_regs_ca_train[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->shadow_regs_ca_train[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->shadow_regs_ca_train[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else if (training & 0xC) {
pmacro_cmd_pad = next_timing->shadow_regs_quse_train[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->shadow_regs_quse_train[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->shadow_regs_quse_train[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->shadow_regs_quse_train[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->shadow_regs_quse_train[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else if (training & 0xF0) {
pmacro_cmd_pad = next_timing->shadow_regs_rdwr_train[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->shadow_regs_rdwr_train[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->shadow_regs_rdwr_train[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->shadow_regs_rdwr_train[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->shadow_regs_rdwr_train[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else {
pmacro_cmd_pad = next_timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->burst_regs[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
}
pmacro_cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & 0xa, 0);
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & 0xf, (100000 / clk) + 1);
ramp_up_wait += 100000;
} else {
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx | 0x8, 0);
}
if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) {
if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
pmacro_cmd_pad |= (EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
pmacro_cmd_pad &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, (100000 / clk) + 1);
ramp_up_wait += 100000;
pmacro_dq_pad |= (EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
pmacro_dq_pad &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0);
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xfe40fe40, 0);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xfe40fe40, (100000 / clk) + 1);
ramp_up_wait += 100000;
}
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xfeedfeed, (100000 / clk) + 1);
ramp_up_wait += 100000;
if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
pmacro_cmd_pad |= (EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, (100000 / clk) + 1);
ramp_up_wait += 100000;
pmacro_dq_pad |= (EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0);
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1, 0);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1, (100000 / clk) + 1);
ramp_up_wait += 100000;
}
ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, (100000 / clk) + 10);
ramp_up_wait += (100000 + (10 * clk));
} else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 | 0x06000600, (100000 / clk) + 1);
ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, (100000 / clk) + 10);
ramp_up_wait += (100000 + 10 * clk);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 | 0x00000600, 0);
ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, 12);
ramp_up_wait += (12 * clk);
}
pmacro_cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, 5);
return ramp_up_wait;
}
static uint32_t apply_periodic_compensation_trimmer(tegra_emc_timing_t* next_timing, uint32_t offset) {
uint32_t temp = 0;
uint32_t next_timing_rate_mhz = next_timing->rate / 1000;
int tree_delta[4] = {0};
int tree_delta_taps[4] = {0};
int new[] = {
TRIM_REG(0, 0, 0, 0),
TRIM_REG(0, 0, 0, 1),
TRIM_REG(0, 0, 1, 2),
TRIM_REG(0, 0, 1, 3),
TRIM_REG(1, 0, 2, 4),
TRIM_REG(1, 0, 2, 5),
TRIM_REG(1, 0, 3, 6),
TRIM_REG(1, 0, 3, 7),
TRIM_REG(0, 1, 0, 0),
TRIM_REG(0, 1, 0, 1),
TRIM_REG(0, 1, 1, 2),
TRIM_REG(0, 1, 1, 3),
TRIM_REG(1, 1, 2, 4),
TRIM_REG(1, 1, 2, 5),
TRIM_REG(1, 1, 3, 6),
TRIM_REG(1, 1, 3, 7)
};
switch (offset) {
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
case EMC_DATA_BRLSHFT_0:
tree_delta[0] = 128 * (next_timing->current_dram_clktree_c0d0u0 - next_timing->trained_dram_clktree_c0d0u0);
tree_delta[1] = 128 * (next_timing->current_dram_clktree_c0d0u1 - next_timing->trained_dram_clktree_c0d0u1);
tree_delta[2] = 128 * (next_timing->current_dram_clktree_c1d0u0 - next_timing->trained_dram_clktree_c1d0u0);
tree_delta[3] = 128 * (next_timing->current_dram_clktree_c1d0u1 - next_timing->trained_dram_clktree_c1d0u1);
tree_delta_taps[0] = (tree_delta[0] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[1] = (tree_delta[1] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[2] = (tree_delta[2] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[3] = (tree_delta[3] * (int)next_timing_rate_mhz) / 1000000;
for (int i = 0; i < 4; i++) {
if ((tree_delta_taps[i] > next_timing->tree_margin) ||
(tree_delta_taps[i] <
(-1 * next_timing->tree_margin))) {
new[i * 2] = new[i * 2] + tree_delta_taps[i];
new[i * 2 + 1] = new[i * 2 + 1] +
tree_delta_taps[i];
}
}
if (offset == EMC_DATA_BRLSHFT_0) {
for (int i = 0; i < 8; i++)
new[i] = new[i] / 64;
} else {
for (int i = 0; i < 8; i++)
new[i] = new[i] % 64;
}
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
case EMC_DATA_BRLSHFT_1:
tree_delta[0] = 128 * (next_timing->current_dram_clktree_c0d1u0 - next_timing->trained_dram_clktree_c0d1u0);
tree_delta[1] = 128 * (next_timing->current_dram_clktree_c0d1u1 - next_timing->trained_dram_clktree_c0d1u1);
tree_delta[2] = 128 * (next_timing->current_dram_clktree_c1d1u0 - next_timing->trained_dram_clktree_c1d1u0);
tree_delta[3] = 128 * (next_timing->current_dram_clktree_c1d1u1 - next_timing->trained_dram_clktree_c1d1u1);
tree_delta_taps[0] = (tree_delta[0] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[1] = (tree_delta[1] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[2] = (tree_delta[2] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[3] = (tree_delta[3] * (int)next_timing_rate_mhz) / 1000000;
for (int i = 0; i < 4; i++) {
if ((tree_delta_taps[i] > next_timing->tree_margin) || (tree_delta_taps[i] < (-1 * next_timing->tree_margin))) {
new[8 + i * 2] = new[8 + i * 2] + tree_delta_taps[i];
new[8 + i * 2 + 1] = new[8 + i * 2 + 1] + tree_delta_taps[i];
}
}
if (offset == EMC_DATA_BRLSHFT_1) {
for (int i = 0; i < 8; i++)
new[i + 8] = new[i + 8] / 64;
} else {
for (int i = 0; i < 8; i++)
new[i + 8] = new[i + 8] % 64;
}
break;
}
switch (offset) {
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
temp = CALC_TEMP(0, 0, 0, 1, 0);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
temp = CALC_TEMP(0, 1, 2, 3, 2);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
temp = CALC_TEMP(0, 2, 4, 5, 4);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
temp = CALC_TEMP(0, 3, 6, 7, 6);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
temp = CALC_TEMP(1, 0, 0, 1, 8);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
temp = CALC_TEMP(1, 1, 2, 3, 10);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
temp = CALC_TEMP(1, 2, 4, 5, 12);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
temp = CALC_TEMP(1, 3, 6, 7, 14);
break;
case EMC_DATA_BRLSHFT_0:
temp = ((new[0] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) |
((new[1] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) |
((new[2] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) |
((new[3] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) |
((new[4] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) |
((new[5] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) |
((new[6] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) |
((new[7] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK);
break;
case EMC_DATA_BRLSHFT_1:
temp = ((new[8] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) |
((new[9] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) |
((new[10] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) |
((new[11] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) |
((new[12] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) |
((new[13] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) |
((new[14] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) |
((new[15] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK);
break;
default:
break;
}
return temp;
}
static uint32_t update_clock_tree_delay(tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, uint32_t dram_dev_num, uint32_t channel_mode, int type) {
uint32_t mrr_req = 0, mrr_data = 0;
uint32_t temp0_0 = 0, temp0_1 = 0, temp1_0 = 0, temp1_1 = 0;
int tdel = 0, tmdel = 0, adel = 0;
uint32_t cval;
uint32_t current_timing_rate_mhz = (current_timing->rate / 1000);
uint32_t next_timing_rate_mhz = (next_timing->rate / 1000);
bool dvfs_pt1 = (type == DVFS_PT1);
bool training_pt1 = (type == TRAINING_PT1);
bool dvfs_update = (type == DVFS_UPDATE);
bool training_update = (type == TRAINING_UPDATE);
bool periodic_training_update = (type == PERIODIC_TRAINING_UPDATE);
/*
* Dev0 MSB.
*/
if (dvfs_pt1 || training_pt1 || periodic_training_update) {
mrr_req = ((2 << EMC_MRR_DEV_SEL_SHIFT) | (19 << EMC_MRR_MA_SHIFT));
emc_write(mrr_req, EMC_MRR);
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC);
if (channel_mode == DUAL_CHANNEL)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC1);
mrr_data = ((emc_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp0_0 = ((mrr_data & 0xff) << 8);
temp0_1 = (mrr_data & 0xff00);
if (channel_mode == DUAL_CHANNEL) {
mrr_data = ((emc1_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp1_0 = ((mrr_data & 0xff) << 8);
temp1_1 = (mrr_data & 0xff00);
}
/*
* Dev0 LSB.
*/
mrr_req = ((mrr_req & ~EMC_MRR_MA_MASK) | (18 << EMC_MRR_MA_SHIFT));
emc_write(mrr_req, EMC_MRR);
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC);
if (channel_mode == DUAL_CHANNEL)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC1);
mrr_data = ((emc_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp0_0 |= (mrr_data & 0xff);
temp0_1 |= ((mrr_data & 0xff00) >> 8);
if (channel_mode == DUAL_CHANNEL) {
mrr_data = ((emc1_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp1_0 |= (mrr_data & 0xff);
temp1_1 |= ((mrr_data & 0xff00) >> 8);
}
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp0_0));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C0D0U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D0U0);
else if (training_update)
__AVERAGE_WRITE_PTFV(C0D0U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D0U0, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = (next_timing->current_dram_clktree_c0d0u0 - __MOVAVG_AC(next_timing, C0D0U0));
tmdel = (tdel < 0) ? -1 * tdel : tdel;
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c0d0u0 = __MOVAVG_AC(next_timing, C0D0U0);
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) /
(current_timing_rate_mhz * 2 * temp0_1));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C0D0U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D0U1);
else if (training_update)
__AVERAGE_WRITE_PTFV(C0D0U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D0U1, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c0d0u1 - __MOVAVG_AC(next_timing, C0D0U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c0d0u1 = __MOVAVG_AC(next_timing, C0D0U1);
}
if (channel_mode == DUAL_CHANNEL) {
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp1_0));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C1D0U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D0U0);
else if (training_update)
__AVERAGE_WRITE_PTFV(C1D0U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D0U0, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c1d0u0 - __MOVAVG_AC(next_timing, C1D0U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c1d0u0 = __MOVAVG_AC(next_timing, C1D0U0);
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp1_1));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C1D0U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D0U1);
else if (training_update)
__AVERAGE_WRITE_PTFV(C1D0U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D0U1, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c1d0u1 - __MOVAVG_AC(next_timing, C1D0U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c1d0u1 = __MOVAVG_AC(next_timing, C1D0U1);
}
}
if (dram_dev_num != TWO_RANK)
return adel;
/*
* Dev1 MSB.
*/
if (dvfs_pt1 || training_pt1 || periodic_training_update) {
mrr_req = ((1 << EMC_MRR_DEV_SEL_SHIFT) | (19 << EMC_MRR_MA_SHIFT));
emc_write(mrr_req, EMC_MRR);
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC);
if (channel_mode == DUAL_CHANNEL)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC1);
mrr_data = ((emc_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp0_0 = ((mrr_data & 0xff) << 8);
temp0_1 = (mrr_data & 0xff00);
if (channel_mode == DUAL_CHANNEL) {
mrr_data = ((emc1_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp1_0 = ((mrr_data & 0xff) << 8);
temp1_1 = (mrr_data & 0xff00);
}
/*
* Dev1 LSB.
*/
mrr_req = ((mrr_req & ~EMC_MRR_MA_MASK) | (18 << EMC_MRR_MA_SHIFT));
emc_write(mrr_req, EMC_MRR);
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC);
if (channel_mode == DUAL_CHANNEL)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC1);
mrr_data = ((emc_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp0_0 |= (mrr_data & 0xff);
temp0_1 |= ((mrr_data & 0xff00) >> 8);
if (channel_mode == DUAL_CHANNEL) {
mrr_data = ((emc1_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp1_0 |= (mrr_data & 0xff);
temp1_1 |= ((mrr_data & 0xff00) >> 8);
}
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp0_0));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C0D1U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D1U0);
else if (training_update)
__AVERAGE_WRITE_PTFV(C0D1U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D1U0, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c0d1u0 - __MOVAVG_AC(next_timing, C0D1U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c0d1u0 = __MOVAVG_AC(next_timing, C0D1U0);
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp0_1));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C0D1U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D1U1);
else if (training_update)
__AVERAGE_WRITE_PTFV(C0D1U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D1U1, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c0d1u1 - __MOVAVG_AC(next_timing, C0D1U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c0d1u1 = __MOVAVG_AC(next_timing, C0D1U1);
}
if (channel_mode == DUAL_CHANNEL) {
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp1_0));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C1D1U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D1U0);
else if (training_update)
__AVERAGE_WRITE_PTFV(C1D1U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D1U0, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c1d1u0 - __MOVAVG_AC(next_timing, C1D1U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c1d1u0 = __MOVAVG_AC(next_timing, C1D1U0);
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp1_1));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C1D1U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D1U1);
else if (training_update)
__AVERAGE_WRITE_PTFV(C1D1U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D1U1, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c1d1u1 - __MOVAVG_AC(next_timing, C1D1U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c1d1u1 = __MOVAVG_AC(next_timing, C1D1U1);
}
}
if (training_update) {
next_timing->trained_dram_clktree_c0d0u0 = next_timing->current_dram_clktree_c0d0u0;
next_timing->trained_dram_clktree_c0d0u1 = next_timing->current_dram_clktree_c0d0u1;
next_timing->trained_dram_clktree_c0d1u0 = next_timing->current_dram_clktree_c0d1u0;
next_timing->trained_dram_clktree_c0d1u1 = next_timing->current_dram_clktree_c0d1u1;
next_timing->trained_dram_clktree_c1d0u0 = next_timing->current_dram_clktree_c1d0u0;
next_timing->trained_dram_clktree_c1d0u1 = next_timing->current_dram_clktree_c1d0u1;
next_timing->trained_dram_clktree_c1d1u0 = next_timing->current_dram_clktree_c1d1u0;
next_timing->trained_dram_clktree_c1d1u1 = next_timing->current_dram_clktree_c1d1u1;
}
return adel;
}
static void reset_dram_clktree_values(tegra_emc_timing_t *table) {
#define __RESET_CLKTREE(TBL, C, D, U) \
TBL->current_dram_clktree_c ## C ## d ## D ## u ## U = \
TBL->trained_dram_clktree_c ## C ## d ## D ## u ## U
__RESET_CLKTREE(table, 0, 0, 0);
__RESET_CLKTREE(table, 0, 0, 1);
__RESET_CLKTREE(table, 1, 0, 0);
__RESET_CLKTREE(table, 1, 0, 1);
__RESET_CLKTREE(table, 1, 1, 0);
__RESET_CLKTREE(table, 1, 1, 1);
}
static uint32_t periodic_compensation_handler(tegra_emc_timing_t *current_timing, tegra_emc_timing_t *next_timing, uint32_t dram_dev_num, uint32_t channel_mode, int type) {
#define __COPY_EMA(nt, lt, dev) \
({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \
(nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
uint32_t adel = 0;
uint32_t samples = next_timing->ptfv_list[PTFV_DVFS_SAMPLES_INDEX];
uint32_t samples_write = next_timing->ptfv_list[PTFV_WRITE_SAMPLES_INDEX];
uint32_t delay = 2 + (1000 * actual_osc_clocks(current_timing->run_clocks) / current_timing->rate);
if (!next_timing->periodic_training)
return 0;
if (type == DVFS_SEQUENCE) {
if (current_timing->periodic_training && (next_timing->ptfv_list[PTFV_CONFIG_CTRL_INDEX] & PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) {
/*
* If the previous frequency was using periodic
* calibration then we can reuse the previous
* frequencies EMA data.
*/
__COPY_EMA(next_timing, current_timing, C0D0U0);
__COPY_EMA(next_timing, current_timing, C0D0U1);
__COPY_EMA(next_timing, current_timing, C1D0U0);
__COPY_EMA(next_timing, current_timing, C1D0U1);
__COPY_EMA(next_timing, current_timing, C0D1U0);
__COPY_EMA(next_timing, current_timing, C0D1U1);
__COPY_EMA(next_timing, current_timing, C1D1U0);
__COPY_EMA(next_timing, current_timing, C1D1U1);
} else {
/* Reset the EMA.*/
__MOVAVG(next_timing, C0D0U0) = 0;
__MOVAVG(next_timing, C0D0U1) = 0;
__MOVAVG(next_timing, C1D0U0) = 0;
__MOVAVG(next_timing, C1D0U1) = 0;
__MOVAVG(next_timing, C0D1U0) = 0;
__MOVAVG(next_timing, C0D1U1) = 0;
__MOVAVG(next_timing, C1D1U0) = 0;
__MOVAVG(next_timing, C1D1U1) = 0;
for (int i = 0; i < samples; i++) {
start_periodic_compensation();
udelay(delay);
/*
* Generate next sample of data.
*/
adel = update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, DVFS_PT1);
}
}
adel = update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, DVFS_UPDATE);
} else if (type == WRITE_TRAINING_SEQUENCE) {
/* Reset the EMA.*/
__MOVAVG(next_timing, C0D0U0) = 0;
__MOVAVG(next_timing, C0D0U1) = 0;
__MOVAVG(next_timing, C1D0U0) = 0;
__MOVAVG(next_timing, C1D0U1) = 0;
__MOVAVG(next_timing, C0D1U0) = 0;
__MOVAVG(next_timing, C0D1U1) = 0;
__MOVAVG(next_timing, C1D1U0) = 0;
__MOVAVG(next_timing, C1D1U1) = 0;
for (int i = 0; i < samples_write; i++) {
start_periodic_compensation();
udelay(delay);
/*
* Generate next sample of data.
*/
update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, TRAINING_PT1);
}
adel = update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, TRAINING_UPDATE);
} else if (type == PERIODIC_TRAINING_SEQUENCE) {
start_periodic_compensation();
udelay(delay);
adel = update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, PERIODIC_TRAINING_UPDATE);
}
return adel;
}
static void set_over_temp_timing(tegra_emc_timing_t *next_timing, unsigned long state) {
#define REFRESH_X2 1
#define REFRESH_X4 2
#define REFRESH_SPEEDUP(val, speedup) \
(val = ((val) & 0xFFFF0000) | (((val) & 0xFFFF) >> (speedup)))
uint32_t ref = next_timing->burst_regs[EMC_REFRESH_INDEX];
uint32_t pre_ref = next_timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
uint32_t dsr_cntrl = next_timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
switch (state) {
case TEGRA_DRAM_OVER_TEMP_NONE:
case TEGRA_DRAM_OVER_TEMP_THROTTLE:
break;
case TEGRA_DRAM_OVER_TEMP_REFRESH_X2:
REFRESH_SPEEDUP(ref, REFRESH_X2);
REFRESH_SPEEDUP(pre_ref, REFRESH_X2);
REFRESH_SPEEDUP(dsr_cntrl, REFRESH_X2);
break;
case TEGRA_DRAM_OVER_TEMP_REFRESH_X4:
REFRESH_SPEEDUP(ref, REFRESH_X4);
REFRESH_SPEEDUP(pre_ref, REFRESH_X4);
REFRESH_SPEEDUP(dsr_cntrl, REFRESH_X4);
break;
default:
return;
}
emc_write(ref, burst_regs_off[EMC_REFRESH_INDEX]);
emc_write(pre_ref, burst_regs_off[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
emc_write(dsr_cntrl, burst_regs_off[EMC_DYN_SELF_REF_CONTROL_INDEX]);
}
static void change_dll_src(tegra_emc_timing_t* next_timing, uint32_t clk_src_emc_to) {
volatile tegra_car_t *car = car_get_regs();
uint32_t emc_2x_clk_src_to = (clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
uint32_t val = (((((next_timing->dll_clk_src & 0x1FFFFFFF) | (emc_2x_clk_src_to << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)) & 0xFFFFFF00) | (clk_src_emc_to & 0xFF)) & 0xFFFFF3FF);
/* Clock source is PLLMB_UD */
if (emc_2x_clk_src_to == TEGRA_EMC_SRC_PLLMB_UD)
val |= 0x400;
else if (emc_2x_clk_src_to != TEGRA_EMC_SRC_PLLM_UD) /* Clock source is not PLLM_UD */
val |= 0x800;
/* Set EMC_DLL_CLK_SRC, DDLL_CLK_SEL and EMC_DLL_CLK_DIVISOR */
car->clk_source_emc_dll = val;
/* Clear and set CLK_ENB_EMC_DLL */
uint32_t clk_enb_emc_dll = ((car->clk_out_enb_x & 0xFFFFBFFF) | ((next_timing->clk_out_enb_x_0_clk_enb_emc_dll & 1) << 14));
car->clk_out_enb_x = clk_enb_emc_dll;
}
static uint32_t dll_prelock(tegra_emc_timing_t* next_timing, bool dvfs_with_training, uint32_t clk_src_emc_to) {
/* Check for dual channel LPDDR4 */
bool dual_channel_lpddr4_case = ((emc_read(EMC_FBIO_CFG7) & EMC_FBIO_CFG7_CH0_ENABLE) & (emc_read(EMC_FBIO_CFG7) & EMC_FBIO_CFG7_CH1_ENABLE));
uint32_t emc_dig_dll_status = 0;
uint32_t emc_cfg_dig_dll = (emc_read(EMC_CFG_DIG_DLL) & ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK);
emc_cfg_dig_dll |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT);
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK;
emc_cfg_dig_dll |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
/* Update EMC_CFG_DIG_DLL_0 */
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is cleared for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
/* Wait until CFG_DLL_EN is cleared for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
}
/* Manual configuration (superseded). */
/*
uint32_t emc_dll_cfg_0 = emc_read(EMC_DLL_CFG_0);
emc_dll_cfg_0 &= 0xDF00000F;
emc_dll_cfg_0 |= 0x1FA340AF;
emc_write(emc_dll_cfg_0, EMC_DLL_CFG_0);
*/
emc_write(next_timing->burst_regs[EMC_DLL_CFG_0_INDEX], EMC_DLL_CFG_0);
emc_write(next_timing->burst_regs[EMC_DLL_CFG_1_INDEX], EMC_DLL_CFG_1);
/* Manual configuration (superseded). */
/*
uint32_t ddllcal_ctrl_start_trim_val = 0;
if ((next_timing->rate >= 400000) && (next_timing->rate < 600000))
ddllcal_ctrl_start_trim_val = 150;
else if ((next_timing->rate >= 600000) && (next_timing->rate < 800000))
ddllcal_ctrl_start_trim_val = 100;
else if ((next_timing->rate >= 800000) && (next_timing->rate < 1000000))
ddllcal_ctrl_start_trim_val = 70;
else if ((next_timing->rate >= 1000000) && (next_timing->rate < 1200000))
ddllcal_ctrl_start_trim_val = 30;
else
ddllcal_ctrl_start_trim_val = 20;
uint32_t emc_dll_cfg_1 = emc_read(EMC_DLL_CFG_1);
emc_dll_cfg_1 &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK;
emc_dll_cfg_1 |= ddllcal_ctrl_start_trim_val;
emc_write(emc_dll_cfg_1, EMC_DLL_CFG_1);
*/
/* Configure the clock and reset controller for EMC DLL */
change_dll_src(next_timing, clk_src_emc_to);
/* Enable DLL */
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is set for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
/* Wait until CFG_DLL_EN is set for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
}
/* Wait until DLL_PRIV_UPDATED or DLL_LOCK have been cleared */
do {
emc_dig_dll_status = emc_read(EMC_DIG_DLL_STATUS);
} while (!(emc_dig_dll_status & EMC_DIG_DLL_STATUS_DLL_LOCK) || !(emc_dig_dll_status & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED));
if (dvfs_with_training) {
/* Set WRITE_MUX to ACTIVE */
emc_set_shadow_bypass(ACTIVE);
/* Disable DLL */
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Set WRITE_MUX to ASSEMBLY */
emc_set_shadow_bypass(ASSEMBLY);
/* Wait until CFG_DLL_EN is cleared for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
/* Wait until CFG_DLL_EN is cleared for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
}
}
/* Return the DLL_OUT value */
return (emc_read(EMC_DIG_DLL_STATUS) & EMC_DIG_DLL_STATUS_DLL_OUT_MASK);
}
static void dll_disable(bool dual_channel_lpddr4_case) {
/* Disable DLL */
uint32_t emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is cleared for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
/* Wait until CFG_DLL_EN is cleared for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
}
}
static void dll_enable(bool dual_channel_lpddr4_case) {
/* Enable DLL */
uint32_t emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is set for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
/* Wait until CFG_DLL_EN is set for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
}
}
static void dll_enable_stall(bool dual_channel_lpddr4_case) {
/* Enable DLL */
uint32_t emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
emc_cfg_dig_dll = (emc_cfg_dig_dll & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is set for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
/* Wait until CFG_DLL_EN is set for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
}
}
static bool test_clk_ratio(uint32_t rate_to, uint32_t clk_src_emc_to, uint32_t rate_from, uint32_t clk_src_emc_from) {
volatile tegra_car_t *car = car_get_regs();
uint32_t emc_2x_clk_src = (car->clk_source_emc >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
uint32_t post_div = 0;
if ((emc_2x_clk_src == TEGRA_EMC_SRC_PLLM) || (emc_2x_clk_src == TEGRA_EMC_SRC_PLLM_UD)) {
post_div = ((car->pllm_base >> 0x14) & 0x1F);
} else if ((emc_2x_clk_src == TEGRA_EMC_SRC_PLLMB_UD) || (emc_2x_clk_src == TEGRA_EMC_SRC_PLLMB)) {
post_div = ((car->pllmb_base >> 0x14) & 0x1F);
}
/* Bad post divider value */
if (post_div > 0x05)
return false;
uint32_t emc_2x_clk_src_from = (clk_src_emc_from >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
uint32_t emc_2x_clk_src_to = (clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
uint8_t emc_2x_clk_div_from = (clk_src_emc_from & 0xFF);
uint8_t emc_2x_clk_div_to = (clk_src_emc_to & 0xFF);
if (emc_2x_clk_src_from <= TEGRA_EMC_SRC_PLLMB_UD)
emc_2x_clk_div_from = 0;
if (emc_2x_clk_src_to <= TEGRA_EMC_SRC_PLLMB_UD)
emc_2x_clk_div_to = 0;
/* Clock sources are different and one of them is CLK_M */
if ((emc_2x_clk_src_to != emc_2x_clk_src_from)
&& ((emc_2x_clk_src_to & 0xFFFFFFFB)
|| (emc_2x_clk_src_from & 0xFFFFFFFB)))
return true;
float val_to = (double)rate_to * ((double)((emc_2x_clk_div_to >> 1) + 1) + (double)(emc_2x_clk_div_to & 1) * 0.5) * (double)(post_div + 1);
float val_from = (double)rate_from * ((double)((emc_2x_clk_div_from >> 1) + 1) + (double)(emc_2x_clk_div_from & 1) * 0.5) * (double)(post_div + 1);
float ratio = (val_from / val_to);
if ((ratio > 1.01f) || (ratio < 0.99f))
return true;
return false;
}
static uint32_t set_pll(uint32_t rate_to, uint32_t rate_osc, uint32_t clk_src_emc_to, bool is_pllmb) {
volatile tegra_car_t *car = car_get_regs();
static const pll_cfg_t pll_vals[] = {
{0xB71B00, 0x2FAF0800, 0x42, 0x01, 0x00}, /* 800Mhz rate with 12Mhz oscillator (unsupported). */
{0xC65D40, 0x2FAF0800, 0x3D, 0x01, 0x00}, /* 800Mhz rate with 13Mhz oscillator (unsupported). */
{0x249F000, 0x11BD0400, 0x5D, 0x04, 0x02}, /* 297.6Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x17D78400, 0x7D, 0x04, 0x02}, /* 400Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x18519600, 0x55, 0x04, 0x01}, /* 408Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x1FC1E200, 0x6F, 0x04, 0x01}, /* 532.8Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x27AC4000, 0x68, 0x03, 0x01}, /* 665.6Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x2FAF0800, 0x7D, 0x03, 0x01}, /* 800Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x3780FC00, 0x61, 0x04, 0x00}, /* 931.2Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x3F83C400, 0x6F, 0x04, 0x00}, /* 1065.6Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x47868C00, 0x7D, 0x04, 0x00}, /* 1200Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x4F588000, 0x68, 0x03, 0x00}, /* 1331.2Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x56F9A000, 0x4C, 0x02, 0x00}, /* 1459.2Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x5F5E1000, 0x7D, 0x03, 0x00}, /* 1600Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x6F01F800, 0x61, 0x02, 0x00}, /* 1862.4Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x7F078800, 0x6F, 0x02, 0x00}, /* 2131.2Mhz rate with 38.4Mhz oscillator. */
{0x00, 0x00, 0x00, 0x00, 0x00} /* Dummy entry. */
};
uint32_t rate_to_hz = (rate_to * 1000);
uint32_t rate_osc_hz = (rate_osc * 1000);
int freq_idx = 0;
for (int i = 0; pll_vals[i].osc_freq; i++) {
if ((rate_osc_hz == pll_vals[i].osc_freq) && (rate_to_hz == pll_vals[i].out_freq)) {
freq_idx = i;
break;
}
}
uint32_t res = clk_src_emc_to;
/* Failed to find the PLL values */
if (!pll_vals[freq_idx].osc_freq)
return res;
uint32_t feedback_div = pll_vals[freq_idx].feedback_div;
uint32_t input_div = pll_vals[freq_idx].input_div;
uint32_t post_div = pll_vals[freq_idx].post_div;
if (is_pllmb) {
/* Set PLLMB_DIVM, PLLMB_DIVN and PLLMB_DIVP */
car->pllmb_base = (input_div | (feedback_div << 0x08) | ((post_div & 0x1F) << 0x14));
/* Set PLLMB_ENABLE */
car->pllmb_base |= 0x40000000;
/* Clock source is PLLM_UD */
if ((clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT) == TEGRA_EMC_SRC_PLLM_UD)
res = ((clk_src_emc_to & 0x1FFFFFFF) | (TEGRA_EMC_SRC_PLLMB_UD << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
else if ((clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT) == TEGRA_EMC_SRC_PLLM) /* Clock source is PLLM_OUT0 */
res = (clk_src_emc_to | (TEGRA_EMC_SRC_PLLMB << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
while (!(car->pllmb_base & 0x8000000)) {
/* Wait for PLLMB_LOCK to be set */
}
} else {
/* Set PLLM_DIVM, PLLM_DIVN and PLLM_DIVP */
car->pllm_base = (input_div | (feedback_div << 0x08) | ((post_div & 0x1F) << 0x14));
/* Set PLLM_EN_LCKDET */
car->pllm_misc2 |= 0x10;
/* Set PLLM_ENABLE */
car->pllm_base |= 0x40000000;
/* Clock source is PLLM_UD */
if ((clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT) == TEGRA_EMC_SRC_PLLM_UD)
res = ((clk_src_emc_to & 0x1FFFFFFF) | (TEGRA_EMC_SRC_PLLM_UD << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
while (!(car->pllm_base & 0x8000000)) {
/* Wait for PLLM_LOCK to be set */
}
}
return res;
}
static void set_clock(tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, uint32_t training, uint32_t next_clk_src) {
volatile tegra_car_t *car = car_get_regs();
/* Extract training values */
bool train_ca = (training & 0x01);
bool train_ca_vref = (training & 0x02);
bool train_quse = (training & 0x04);
bool train_quse_vref = (training & 0x08);
bool train_wr = (training & 0x10);
bool train_wr_vref = (training & 0x20);
bool train_rd = (training & 0x40);
bool train_rd_vref = (training & 0x80);
bool train_swap_rank = (training & 0x100);
bool train_self_refresh = (training & 0x200);
/* Check if we should do training. */
bool dvfs_with_training = (training & 0xF7);
bool skip_zqcal = false;
bool compensate_trimmer_applicable = false;
uint32_t zqcal_before_cc_cutoff = 2400; /* In picoseconds */
int zq_latch_dvfs_wait_time;
uint32_t mr13_catr_enable;
uint32_t mr13_flip_fspwr;
uint32_t mr13_flip_fspop;
int next_push, next_dq_e_ivref, next_dqs_e_ivref;
uint32_t zq_wait_long;
uint32_t zq_wait_short;
uint32_t tRTM;
uint32_t RP_war;
uint32_t R2P_war;
uint32_t TRPab_war;
int nRTP;
uint32_t deltaTWATM;
uint32_t W2P_war;
uint32_t tRPST;
uint32_t mrw_req;
uint32_t adel = 0;
uint32_t next_timing_rate_mhz = next_timing->rate / 1000;
/* Set some common values needed. */
int dram_type = emc_read(EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;
int dram_dev_num = ((mc_read(MC_EMEM_ADR_CFG) & 1) + 1);
bool shared_zq_resistor = ((current_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] >> 31) & 1);
bool channel_mode = ((current_timing->burst_regs[EMC_FBIO_CFG7_INDEX] >> 2) & 1);
bool is_lpddr3 = (dram_type == DRAM_TYPE_LPDDR2) && ((next_timing->burst_regs[EMC_FBIO_CFG5_INDEX] >> 25) & 1);
bool opt_zcal_en_cc = ((next_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX] && !current_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX]) || (dram_type == DRAM_TYPE_LPDDR4));
bool opt_war_200024907 = (dram_type == DRAM_TYPE_LPDDR4);
bool opt_do_sw_qrst = false;
bool opt_cc_short_zcal = true;
bool opt_short_zcal = true;
bool save_restore_clkstop_pd = true;
uint32_t opt_dll_mode = (dram_type == DRAM_TYPE_DDR3) ? get_dll_state(next_timing) : DLL_OFF;
uint32_t opt_dvfs_mode = MAN_SR;
uint32_t emc_auto_cal_config = emc_read(EMC_AUTO_CAL_CONFIG);
/* In picoseconds. */
uint32_t source_clock_period = 1000000000 / current_timing->rate;
uint32_t destination_clock_period = 1000000000 / next_timing->rate;
uint32_t tFC_lpddr4 = 1000 * next_timing->dram_timings[T_FC_LPDDR4];
uint32_t tZQCAL_lpddr4 = 1000000;
int tZQCAL_lpddr4_fc_adj = (source_clock_period > zqcal_before_cc_cutoff) ? tZQCAL_lpddr4 / destination_clock_period : (tZQCAL_lpddr4 - tFC_lpddr4) / destination_clock_period;
g_fsp_for_next_freq = !g_fsp_for_next_freq;
uint32_t emc_dbg_o = emc_read(EMC_DBG);
uint32_t emc_pin_o = emc_read(EMC_PIN);
uint32_t emc_cfg_pipe_clk_o = emc_read(EMC_CFG_PIPE_CLK);
uint32_t emc_dbg = emc_dbg_o;
uint32_t emc_cfg = next_timing->burst_regs[EMC_CFG_INDEX];
uint32_t emc_sel_dpd_ctrl = next_timing->emc_sel_dpd_ctrl;
emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD);
emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN | EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN | EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN | EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN | EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN);
/* Step 1:
* Pre DVFS SW sequence.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 1\n");
/* Step 1.1: Disable DLL. */
uint32_t tmp = emc_read(EMC_CFG_DIG_DLL);
tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(tmp, EMC_CFG_DIG_DLL);
/* Request a timing update. */
emc_timing_update(channel_mode);
/* Wait for DLL to be disabled. */
wait_for_update(EMC_CFG_DIG_DLL, EMC_CFG_DIG_DLL_CFG_DLL_EN, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_CFG_DIG_DLL, EMC_CFG_DIG_DLL_CFG_DLL_EN, false, REG_EMC1);
/* Step 1.2: Disable AUTOCAL. */
emc_auto_cal_config = next_timing->emc_auto_cal_config;
uint32_t auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE;
emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START;
emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL;
emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL;
emc_auto_cal_config |= auto_cal_en;
emc_write(emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
/* Step 1.3: Disable other power features. */
emc_set_shadow_bypass(ACTIVE);
emc_write(emc_cfg, EMC_CFG);
emc_write(emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
emc_set_shadow_bypass(ASSEMBLY);
/* Skip this if dvfs_with_training is set. */
if (!dvfs_with_training && next_timing->periodic_training) {
if (dram_dev_num == TWO_RANK) {
/* Wait for DRAM to get out of power down. */
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, false, REG_EMC1);
} else {
wait_for_update(EMC_EMC_STATUS, 0x10, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, 0x10, false, REG_EMC1);
}
/* Wait for DRAM to get out of self refresh. */
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, false, REG_EMC1);
/* Reset all clock tree values. */
reset_dram_clktree_values(next_timing);
/* Do DVFS_SEQUENCE. */
adel = periodic_compensation_handler(current_timing, next_timing, dram_dev_num, channel_mode, DVFS_SEQUENCE);
/* Check if we should use compensate trimmer. */
compensate_trimmer_applicable = next_timing->periodic_training && ((adel * 128 * next_timing_rate_mhz) / 1000000) > next_timing->tree_margin;
}
emc_write(EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS);
emc_set_shadow_bypass(ACTIVE);
emc_write(emc_cfg, EMC_CFG);
emc_write(emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
emc_write(emc_cfg_pipe_clk_o | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON, EMC_CFG_PIPE_CLK);
emc_write(next_timing->emc_fdpd_ctrl_cmd_no_ramp & ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE, EMC_FDPD_CTRL_CMD_NO_RAMP);
uint32_t bg_regulator_mode_change = ((next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^ (current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) || ((next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^ (current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD));
uint32_t enable_bg_regulator = (next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0;
/* Check if we need to change BG the regulator. */
if (bg_regulator_mode_change) {
if (enable_bg_regulator)
emc_write(current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, EMC_PMACRO_BG_BIAS_CTRL_0);
else
emc_write(current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, EMC_PMACRO_BG_BIAS_CTRL_0);
}
/* Check if we need to turn on VREF generator. */
if ((((!current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF)) &&
((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF))) ||
((!(current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF)) &&
((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF))))
{
uint32_t pad_tx_ctrl = next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
uint32_t last_pad_tx_ctrl = current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
next_dqs_e_ivref = pad_tx_ctrl & EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF;
next_dq_e_ivref = pad_tx_ctrl & EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF;
next_push = (last_pad_tx_ctrl & ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF & ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) | next_dq_e_ivref | next_dqs_e_ivref;
emc_write(next_push, EMC_PMACRO_DATA_PAD_TX_CTRL);
udelay(1);
} else if (bg_regulator_mode_change) {
udelay(1);
}
emc_set_shadow_bypass(ASSEMBLY);
/* Step 2:
* Prelock the DLL.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 2\n");
if (next_timing->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) {
dll_prelock(next_timing, dvfs_with_training, next_clk_src);
} else {
change_dll_src(next_timing, next_clk_src);
dll_disable(channel_mode);
}
/* Step 3:
* Prepare autocal for the clock change.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 3\n");
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2);
emc_write(next_timing->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3);
emc_write(next_timing->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4);
emc_write(next_timing->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5);
emc_write(next_timing->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6);
emc_write(next_timing->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7);
emc_write(next_timing->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8);
emc_set_shadow_bypass(ASSEMBLY);
emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START | auto_cal_en);
emc_write(emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
/* Step 4:
* Update EMC_CFG.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 4\n");
if ((source_clock_period > 50000) && (dram_type == DRAM_TYPE_LPDDR4))
ccfifo_write(EMC_SELF_REF, 1, 0);
else
emc_write(next_timing->emc_cfg_2, EMC_CFG_2);
/* Step 5:
* Prepare reference variables for ZQCAL regs.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 5\n");
uint32_t emc_zcal_interval = current_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX];
emc_zcal_interval &= 0xFF000000;
uint32_t emc_zcal_wait_cnt_old = current_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
uint32_t emc_zcal_wait_cnt_new = next_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
emc_zcal_wait_cnt_old &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
emc_zcal_wait_cnt_new &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
if (dram_type == DRAM_TYPE_LPDDR4)
zq_wait_long = max((uint32_t)1, div_o3(1000000, destination_clock_period));
else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
zq_wait_long = max(next_timing->min_mrs_wait, div_o3(360000, destination_clock_period)) + 4;
else if (dram_type == DRAM_TYPE_DDR3)
zq_wait_long = max((uint32_t)256, div_o3(320000, destination_clock_period) + 2);
else
zq_wait_long = 0;
if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
zq_wait_short = max(max(next_timing->min_mrs_wait, (uint32_t)6), div_o3(90000, destination_clock_period)) + 4;
else if (dram_type == DRAM_TYPE_DDR3)
zq_wait_short = max((uint32_t)64, div_o3(80000, destination_clock_period)) + 2;
else
zq_wait_short = 0;
/* TODO: Actually use the reference variables. */
(void)zq_wait_long;
(void)zq_wait_short;
/* Step 6:
* Training code.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 6\n");
if ((train_ca || train_ca_vref) && (dram_dev_num == TWO_RANK)) {
emc_write(0x107, EMC_PIN);
}
/* Step 7:
* Program FSP reference registers and send MRWs to new FSPWR.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 7\n");
/* Step 7.1: Bug 200024907 - Patch RP R2P */
if (opt_war_200024907) {
nRTP = 16;
if (source_clock_period >= 1000000/1866) /* 535.91 ps */
nRTP = 14;
if (source_clock_period >= 1000000/1600) /* 625.00 ps */
nRTP = 12;
if (source_clock_period >= 1000000/1333) /* 750.19 ps */
nRTP = 10;
if (source_clock_period >= 1000000/1066) /* 938.09 ps */
nRTP = 8;
deltaTWATM = max_t(uint32_t, div_o3(7500, source_clock_period), 8);
/*
* Originally there was a + .5 in the tRPST calculation.
* However since we can't do FP in the kernel and the tRTM
* computation was in a floating point ceiling function, adding
* one to tRTP should be ok. There is no other source of non
* integer values, so the result was always going to be
* something for the form: f_ceil(N + .5) = N + 1;
*/
tRPST = ((current_timing->emc_mrw & 0x80) >> 7);
tRTM = current_timing->dram_timings[RL] + div_o3(3600, source_clock_period) + max_t(uint32_t, div_o3(7500, source_clock_period), 8) + tRPST + 1 + nRTP;
if (current_timing->burst_regs[EMC_RP_INDEX] < tRTM) {
if (tRTM > (current_timing->burst_regs[EMC_R2P_INDEX] + current_timing->burst_regs[EMC_RP_INDEX])) {
R2P_war = tRTM - current_timing->burst_regs[EMC_RP_INDEX];
RP_war = current_timing->burst_regs[EMC_RP_INDEX];
TRPab_war = current_timing->burst_regs[EMC_TRPAB_INDEX];
if (R2P_war > 63) {
RP_war = R2P_war + current_timing->burst_regs[EMC_RP_INDEX] - 63;
if (TRPab_war < RP_war)
TRPab_war = RP_war;
R2P_war = 63;
}
} else {
R2P_war = current_timing-> burst_regs[EMC_R2P_INDEX];
RP_war = current_timing->burst_regs[EMC_RP_INDEX];
TRPab_war = current_timing->burst_regs[EMC_TRPAB_INDEX];
}
if (RP_war < deltaTWATM) {
W2P_war = current_timing->burst_regs[EMC_W2P_INDEX] + deltaTWATM - RP_war;
if (W2P_war > 63) {
RP_war = RP_war + W2P_war - 63;
if (TRPab_war < RP_war)
TRPab_war = RP_war;
W2P_war = 63;
}
} else {
W2P_war = current_timing->burst_regs[EMC_W2P_INDEX];
}
if ((current_timing->burst_regs[EMC_W2P_INDEX] != W2P_war)
|| (current_timing->burst_regs[EMC_R2P_INDEX] != R2P_war)
|| (current_timing->burst_regs[EMC_RP_INDEX] != RP_war)
|| (current_timing->burst_regs[EMC_TRPAB_INDEX] != TRPab_war))
{
emc_set_shadow_bypass(ACTIVE);
emc_write(RP_war, EMC_RP);
emc_write(R2P_war, EMC_R2P);
emc_write(W2P_war, EMC_W2P);
emc_write(TRPab_war, EMC_TRPAB);
emc_set_shadow_bypass(ASSEMBLY);
udelay(1);
}
}
}
if (!g_fsp_for_next_freq) {
mr13_flip_fspwr = (next_timing->emc_mrw3 & 0xffffff3f) | 0x80;
mr13_flip_fspop = (next_timing->emc_mrw3 & 0xffffff3f) | 0x00;
} else {
mr13_flip_fspwr = (next_timing->emc_mrw3 & 0xffffff3f) | 0x40;
mr13_flip_fspop = (next_timing->emc_mrw3 & 0xffffff3f) | 0xc0;
}
mr13_catr_enable = (mr13_flip_fspwr & 0xFFFFFFFE) | 0x01;
if (dram_dev_num == TWO_RANK) {
if (train_ca || train_ca_vref) {
if (train_swap_rank) {
mr13_flip_fspop = (mr13_flip_fspop & 0x3FFFFFFF) | 0x80000000;
mr13_catr_enable = (mr13_catr_enable & 0x3FFFFFFF)| 0x40000000;
} else {
mr13_flip_fspop = (mr13_flip_fspop & 0x3FFFFFFF) | 0x40000000;
mr13_catr_enable = (mr13_catr_enable & 0x3FFFFFFF) | 0x80000000;
}
} else {
if (train_swap_rank)
mr13_catr_enable = (mr13_catr_enable & 0x3FFFFFFF) | 0x40000000;
else
mr13_catr_enable = (mr13_catr_enable & 0x3FFFFFFF) | 0x80000000;
}
}
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_write(mr13_flip_fspwr, EMC_MRW3);
emc_write(next_timing->emc_mrw, EMC_MRW);
emc_write(next_timing->emc_mrw2, EMC_MRW2);
}
/* Step 8:
* Program the shadow registers.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 8\n");
/* Set burst registers. */
for (int i = 0; i < next_timing->num_burst; i++) {
uint32_t var = 0;
uint32_t wval = 0;
if (!burst_regs_off[i])
continue;
var = burst_regs_off[i];
if (dvfs_with_training) {
if (train_ca || train_ca_vref)
wval = next_timing->shadow_regs_ca_train[i];
else if (train_quse || train_quse_vref)
wval = next_timing->shadow_regs_quse_train[i];
else if (train_wr || train_wr_vref || train_rd || train_rd_vref)
wval = next_timing->shadow_regs_rdwr_train[i];
}
else
wval = next_timing->burst_regs[i];
if (dram_type != DRAM_TYPE_LPDDR4 &&
(var == EMC_MRW6 || var == EMC_MRW7 ||
var == EMC_MRW8 || var == EMC_MRW9 ||
var == EMC_MRW10 || var == EMC_MRW11 ||
var == EMC_MRW12 || var == EMC_MRW13 ||
var == EMC_MRW14 || var == EMC_MRW15 ||
var == EMC_TRAINING_CTRL))
continue;
if (var == EMC_CFG) {
wval &= ~EMC_CFG_DRAM_ACPD;
wval &= ~EMC_CFG_DYN_SELF_REF;
if (dram_type == DRAM_TYPE_LPDDR4) {
wval &= ~EMC_CFG_DRAM_CLKSTOP_SR;
wval &= ~EMC_CFG_DRAM_CLKSTOP_PD;
}
} else if ((var == EMC_MRS_WAIT_CNT)
&& (dram_type == DRAM_TYPE_LPDDR2)
&& opt_zcal_en_cc && !opt_cc_short_zcal && opt_short_zcal) {
wval = (wval & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK <<
EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) |
((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) <<
EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
} else if ((var == EMC_ZCAL_WAIT_CNT)
&& (dram_type == DRAM_TYPE_DDR3)
&& opt_zcal_en_cc && !opt_cc_short_zcal && opt_short_zcal) {
wval = (wval & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK <<
EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) |
((zq_wait_long &
EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) <<
EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
} else if ((var == EMC_ZCAL_INTERVAL) && opt_zcal_en_cc) {
wval = 0; /* EMC_ZCAL_INTERVAL reset value. */
} else if (var == EMC_PMACRO_AUTOCAL_CFG_COMMON) {
wval |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS;
} else if (var == EMC_PMACRO_DATA_PAD_TX_CTRL) {
wval &=
~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
} else if (var == EMC_PMACRO_CMD_PAD_TX_CTRL) {
wval |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
wval &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
} else if (var == EMC_PMACRO_BRICK_CTRL_RFU1) {
wval &= 0xf800f800;
} else if (var == EMC_PMACRO_COMMON_PAD_TX_CTRL) {
wval &= 0xfffffff0;
} else if (var == EMC_TRAINING_CTRL) {
wval |= (train_swap_rank << 14); /* Training only. */
}
emc_write(wval, var);
}
/* Do EMC refresh adjustment here (disabled). */
set_over_temp_timing(next_timing, TEGRA_DRAM_OVER_TEMP_NONE);
if (dram_type == DRAM_TYPE_LPDDR4) {
/* Use the current timing when training. */
if (dvfs_with_training)
mrw_req = (23 << EMC_MRW_MRW_MA_SHIFT) | (current_timing->run_clocks & EMC_MRW_MRW_OP_MASK);
else
mrw_req = (23 << EMC_MRW_MRW_MA_SHIFT) | (next_timing->run_clocks & EMC_MRW_MRW_OP_MASK);
emc_write(mrw_req, EMC_MRW);
}
/* Per channel burst registers. */
for (int i = 0; i < next_timing->num_burst_per_ch; i++) {
if (!burst_regs_per_ch_off[i])
continue;
if (dram_type != DRAM_TYPE_LPDDR4 &&
(burst_regs_per_ch_off[i] == EMC_MRW6 ||
burst_regs_per_ch_off[i] == EMC_MRW7 ||
burst_regs_per_ch_off[i] == EMC_MRW8 ||
burst_regs_per_ch_off[i] == EMC_MRW9 ||
burst_regs_per_ch_off[i] == EMC_MRW10 ||
burst_regs_per_ch_off[i] == EMC_MRW11 ||
burst_regs_per_ch_off[i] == EMC_MRW12 ||
burst_regs_per_ch_off[i] == EMC_MRW13 ||
burst_regs_per_ch_off[i] == EMC_MRW14 ||
burst_regs_per_ch_off[i] == EMC_MRW15))
continue;
/* Filter out second channel if not in DUAL_CHANNEL mode. */
if ((channel_mode != DUAL_CHANNEL) && (burst_regs_per_ch_type[i] >= REG_EMC1))
continue;
emc_write_per_ch(next_timing->burst_reg_per_ch[i], burst_regs_per_ch_type[i], burst_regs_per_ch_off[i]);
}
/* Vref regs. */
for (int i = 0; i < next_timing->vref_num; i++) {
if (!vref_regs_per_ch_off[i])
continue;
if ((channel_mode != DUAL_CHANNEL) && (vref_regs_per_ch_type[i] >= REG_EMC1))
continue;
emc_write_per_ch(next_timing->vref_perch_regs[i], vref_regs_per_ch_type[i], vref_regs_per_ch_off[i]);
}
/* Training regs. */
if (dvfs_with_training) {
for (int i = 0; i < next_timing->training_mod_num; i++) {
if (!training_mod_regs_per_ch_off[i])
continue;
if ((channel_mode != DUAL_CHANNEL) && (training_mod_regs_per_ch_type[i] >= REG_EMC1))
continue;
emc_write_per_ch(next_timing->training_mod_regs[i], training_mod_regs_per_ch_type[i], training_mod_regs_per_ch_off[i]);
}
}
/* Trimmers. */
for (int i = 0; i < next_timing->num_trim; i++) {
uint32_t trim_reg;
if (!trim_regs_off[i])
continue;
trim_reg = trim_regs_off[i];
if (compensate_trimmer_applicable &&
(trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
trim_reg == EMC_DATA_BRLSHFT_0 ||
trim_reg == EMC_DATA_BRLSHFT_1)) {
uint32_t reg = apply_periodic_compensation_trimmer(next_timing, trim_reg);
emc_write(reg, trim_regs_off[i]);
} else {
emc_write(next_timing->trim_regs[i], trim_regs_off[i]);
}
}
/* Per channel trimmers. */
for (int i = 0; i < next_timing->num_trim_per_ch; i++) {
uint32_t trim_reg;
if (!trim_regs_per_ch_off[i])
continue;
if ((channel_mode != DUAL_CHANNEL) && (trim_regs_per_ch_type[i] >= REG_EMC1))
continue;
trim_reg = trim_regs_per_ch_off[i];
if (compensate_trimmer_applicable &&
(trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
trim_reg == EMC_DATA_BRLSHFT_0 ||
trim_reg == EMC_DATA_BRLSHFT_1)) {
uint32_t reg = apply_periodic_compensation_trimmer(next_timing, trim_reg);
emc_write_per_ch(reg, trim_regs_per_ch_type[i], trim_regs_per_ch_off[i]);
} else {
emc_write_per_ch(next_timing->trim_perch_regs[i], trim_regs_per_ch_type[i], trim_regs_per_ch_off[i]);
}
}
if (dvfs_with_training) {
if (train_wr && next_timing->periodic_training && (dram_type == DRAM_TYPE_LPDDR4)) {
periodic_compensation_handler(current_timing, next_timing, dram_dev_num, channel_mode, WRITE_TRAINING_SEQUENCE);
}
} else {
/* Write burst_mc_regs. */
for (int i = 0; i < next_timing->num_mc_regs; i++) {
mc_write(next_timing->burst_mc_regs[i], burst_mc_regs_off[i]);
}
}
/* Registers to be programmed on the faster clock. */
if (!dvfs_with_training && (next_timing->rate < current_timing->rate)) {
for (int i = 0; i < next_timing->num_up_down; i++) {
mc_write(next_timing->la_scale_regs[i], la_scale_regs_off[i]);
}
}
/* Step 9:
* LPDDR4 section A.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 9\n");
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_write(emc_zcal_interval, EMC_ZCAL_INTERVAL);
emc_write(emc_zcal_wait_cnt_new, EMC_ZCAL_WAIT_CNT);
emc_write(emc_dbg_o | (EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY), EMC_DBG);
emc_write(emc_zcal_interval, EMC_ZCAL_INTERVAL);
emc_write(emc_dbg_o, EMC_DBG);
if (dvfs_with_training) {
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX] | EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS, EMC_PMACRO_AUTOCAL_CFG_COMMON);
if (train_ca || train_ca_vref)
emc_write(current_timing->burst_regs[EMC_FBIO_CFG5_INDEX] | EMC_FBIO_CFG5_CMD_BUS_RETURN_TO_ZERO, EMC_FBIO_CFG5);
emc_set_shadow_bypass(ASSEMBLY);
if (channel_mode)
ccfifo_write(EMC_CFG_SYNC, 0, 0);
/* Change CFG_SWAP. */
ccfifo_write(EMC_DBG, ((emc_dbg_o & 0xF3FFFFFF) | 0x4000000), 0);
}
}
/* Step 10:
* LPDDR4 and DDR3 common section.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 10\n");
if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) {
if (dram_type == DRAM_TYPE_LPDDR4)
ccfifo_write(EMC_SELF_REF, 0x101, 0);
else
ccfifo_write(EMC_SELF_REF, 0x1, 0);
if (!(train_ca || train_ca_vref) && (dram_type == DRAM_TYPE_LPDDR4) && (source_clock_period <= zqcal_before_cc_cutoff)) {
ccfifo_write(EMC_MRW3, mr13_flip_fspwr ^ 0x40, 0);
ccfifo_write(EMC_MRW6, (next_timing->burst_regs[EMC_MRW6_INDEX] & 0xFFFF3F3F) | (current_timing->burst_regs[EMC_MRW6_INDEX] & 0x0000C0C0), 0);
ccfifo_write(EMC_MRW14, (next_timing->burst_regs[EMC_MRW14_INDEX] & 0xFFFF0707) | (current_timing->burst_regs[EMC_MRW14_INDEX] & 0x00003838), 0);
if (dram_dev_num == TWO_RANK) {
ccfifo_write(EMC_MRW7, (next_timing->burst_regs[EMC_MRW7_INDEX] & 0xFFFF3F3F) | (current_timing->burst_regs[EMC_MRW7_INDEX] & 0x0000C0C0), 0);
ccfifo_write(EMC_MRW15, (next_timing->burst_regs[EMC_MRW15_INDEX] & 0xFFFF0707) | (current_timing->burst_regs[EMC_MRW15_INDEX] & 0x00003838), 0);
}
if (opt_zcal_en_cc) {
if ((dram_dev_num == ONE_RANK) || shared_zq_resistor)
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
else
ccfifo_write(EMC_ZQ_CAL, EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
}
}
}
emc_dbg = emc_dbg_o;
if (dram_type == DRAM_TYPE_LPDDR4) {
if (dvfs_with_training) {
/* Change CFG_SWAP. */
emc_dbg = ((emc_dbg_o & 0xF3FFFFFF) | 0x4000000 | EMC_DBG_WRITE_ACTIVE_ONLY);
ccfifo_write(EMC_DBG, emc_dbg, 0);
}
if (train_ca || train_ca_vref) {
ccfifo_write(EMC_PMACRO_DATA_RX_TERM_MODE, current_timing->burst_regs[EMC_PMACRO_DATA_RX_TERM_MODE_INDEX] & 0xFFFFFCCC, 0);
if ((dram_dev_num == TWO_RANK) && train_swap_rank) {
ccfifo_write(EMC_MRW3, mr13_flip_fspop | 0x8, (1000 * current_timing->dram_timings[T_RP]) / source_clock_period);
ccfifo_write(EMC_MRW3, mr13_catr_enable | 0x8, 0);
} else {
ccfifo_write(EMC_MRW3, mr13_catr_enable | 0x8, (1000 * current_timing->dram_timings[T_RP]) / source_clock_period);
}
ccfifo_write(EMC_TR_CTRL_0, 0x15A, 0);
ccfifo_write(EMC_INTSTATUS, 0, 1000000 / source_clock_period);
} else {
ccfifo_write(EMC_MRW3, mr13_flip_fspop | 0x8, (1000 * current_timing->dram_timings[T_RP]) / source_clock_period);
ccfifo_write(EMC_INTSTATUS, 0, tFC_lpddr4 / source_clock_period);
}
}
bool ref_b4_sref_en = false;
bool cya_issue_pc_ref = false;
bool cya_allow_ref_cc = false;
if ((dram_type == DRAM_TYPE_LPDDR4) || (opt_dvfs_mode != MAN_SR)) {
uint32_t t = 30 + (cya_allow_ref_cc ? (4000 * current_timing->dram_timings[T_RFC]) + ((1000 * current_timing->dram_timings[T_RP]) / source_clock_period) : 0);
ccfifo_write(EMC_PIN, emc_pin_o & ~(EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE), t);
}
uint32_t ref_delay_mult = 1;
ref_delay_mult += ref_b4_sref_en ? 1 : 0;
ref_delay_mult += cya_allow_ref_cc ? 1 : 0;
ref_delay_mult += cya_issue_pc_ref ? 1 : 0;
uint32_t ref_delay = ref_delay_mult * ((1000 * current_timing->dram_timings[T_RP] / source_clock_period) + (1000 * current_timing->dram_timings[T_RFC] / source_clock_period)) + 20;
/* Step 11:
* Ramp down.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 11\n");
ccfifo_write(EMC_CFG_SYNC, 0, (dram_type == DRAM_TYPE_LPDDR4) ? 0 : ref_delay);
ccfifo_write(EMC_DBG, emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY), 0);
uint32_t ramp_down_wait = dvfs_power_ramp_down(false, current_timing, next_timing, source_clock_period);
/* Step 12:
* Trigger the clock change.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 12\n");
ccfifo_write(EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 1, 0);
if (!dvfs_with_training) {
ccfifo_write(EMC_DBG, (emc_dbg & ~EMC_DBG_WRITE_ACTIVE_ONLY) | EMC_DBG_WRITE_MUX_ACTIVE, 0);
}
/* Step 13:
* Ramp up.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 13\n");
uint32_t ramp_up_wait = dvfs_power_ramp_up(false, current_timing, next_timing, training, destination_clock_period);
ccfifo_write(EMC_DBG, emc_dbg, 0);
/* Step 14:
* Bringup CKE pins.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 14\n");
if ((dram_type == DRAM_TYPE_LPDDR4)) {
uint32_t r = emc_pin_o | EMC_PIN_PIN_CKE;
if (train_ca || train_ca_vref) {
r = emc_pin_o & ~(EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE);
if (dram_dev_num == TWO_RANK) {
if (train_swap_rank)
ccfifo_write(EMC_PIN, r | EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKE, 0);
else
ccfifo_write(EMC_PIN, r | EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB, 0);
}
else
ccfifo_write(EMC_PIN, r, 0);
} else if (dram_dev_num == TWO_RANK) {
ccfifo_write(EMC_PIN, r | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV, 0);
} else {
ccfifo_write(EMC_PIN, r & ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV), 0);
}
}
/* Step 15:
* Calculate zqlatch wait time; has dependency on ramping times.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 15\n");
if (source_clock_period <= zqcal_before_cc_cutoff) {
int t = (int)(ramp_up_wait + ramp_down_wait) / (int)destination_clock_period;
zq_latch_dvfs_wait_time = (int)tZQCAL_lpddr4_fc_adj - t;
} else {
zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj - div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period);
}
if (!(train_ca || train_ca_vref) && (dram_type == DRAM_TYPE_LPDDR4) && opt_zcal_en_cc) {
if (dram_dev_num == ONE_RANK) {
if (source_clock_period > zqcal_before_cc_cutoff) {
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
}
if (!dvfs_with_training) {
ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
ccfifo_write(EMC_SELF_REF, 0x100, 0);
ccfifo_write(EMC_REF, 0, 0);
}
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, max_t(int, 0, zq_latch_dvfs_wait_time));
} else if (shared_zq_resistor) {
if (source_clock_period > zqcal_before_cc_cutoff) {
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
}
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, max_t(int, 0, zq_latch_dvfs_wait_time) + div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, 0);
if (!dvfs_with_training) {
ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, 0);
ccfifo_write(EMC_SELF_REF, 0x100, 0);
ccfifo_write(EMC_REF, 0, 0);
}
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, tZQCAL_lpddr4 / destination_clock_period);
} else {
if (source_clock_period > zqcal_before_cc_cutoff) {
ccfifo_write(EMC_ZQ_CAL, EMC_ZQ_CAL_ZQ_CAL_CMD, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
}
if (!dvfs_with_training) {
ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
ccfifo_write(EMC_SELF_REF, 0x100, 0);
ccfifo_write(EMC_REF, 0, 0);
}
ccfifo_write(EMC_ZQ_CAL, EMC_ZQ_CAL_ZQ_LATCH_CMD, max_t(int, 0, zq_latch_dvfs_wait_time));
}
}
/* WAR: delay for zqlatch */
ccfifo_write(EMC_INTSTATUS, 0, 10);
/* Step 16:
* LPDDR4 Conditional Training Kickoff.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 16\n");
if (dvfs_with_training && (dram_type == DRAM_TYPE_LPDDR4)) {
ccfifo_write(EMC_INTSTATUS, 0, (1020000 / destination_clock_period));
uint32_t train_cmd = 0;
if (train_ca)
train_cmd |= (1 << 1); /* CA */
if (train_ca_vref)
train_cmd |= (1 << 5); /* CA_VREF */
if (train_quse)
train_cmd |= (1 << 4); /* QUSE */
if (train_quse_vref)
train_cmd |= (1 << 8); /* QUSE_VREF */
if (train_wr)
train_cmd |= (1 << 3); /* WR */
if (train_wr_vref)
train_cmd |= (1 << 6); /* WR_VREF */
if (train_rd)
train_cmd |= (1 << 2); /* RD */
if (train_rd_vref)
train_cmd |= (1 << 7); /* RD_VREF */
train_cmd |= (1 << 31); /* GO */
ccfifo_write(EMC_TRAINING_CMD, train_cmd, 0);
if (bg_regulator_mode_change) {
if (enable_bg_regulator)
ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0, current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, 0);
else
ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0, current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, 0);
}
ccfifo_write(EMC_SWITCH_BACK_CTRL, 1, 0);
if (!(train_ca || train_ca_vref) || train_swap_rank) {
ccfifo_write(EMC_MRW3, mr13_flip_fspop ^ 0xC0, 0);
ccfifo_write(EMC_INTSTATUS, 0, (1000000 / destination_clock_period));
}
ccfifo_write(EMC_PIN, emc_pin_o & ~(EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE), 0);
ccfifo_write(EMC_CFG_SYNC, 0, 0);
ccfifo_write(EMC_DBG, emc_dbg | (EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_WRITE_MUX_ACTIVE), 0);
dvfs_power_ramp_down(true, current_timing, next_timing, destination_clock_period);
ccfifo_write(EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 1, 0);
ccfifo_write(EMC_DBG, (emc_dbg & ~EMC_DBG_WRITE_ACTIVE_ONLY) | EMC_DBG_WRITE_MUX_ACTIVE, 0);
dvfs_power_ramp_up(true, current_timing, next_timing, training, source_clock_period);
ccfifo_write(EMC_DBG, emc_dbg, 0);
if (dram_dev_num == TWO_RANK)
ccfifo_write(EMC_PIN, emc_pin_o | (EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE), 0);
else
ccfifo_write(EMC_PIN, (emc_pin_o & ~(EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE)) | EMC_PIN_PIN_CKE, 0);
if (train_ca || train_ca_vref) {
ccfifo_write(EMC_TR_CTRL_0, 0x4A, (200000 / source_clock_period));
ccfifo_write(EMC_TR_CTRL_0, 0x40, (1000000 / source_clock_period));
ccfifo_write(EMC_MRW3, mr13_catr_enable & 0xFFFFFFFE, 0);
ccfifo_write(EMC_INTSTATUS, 0, (1000000 / source_clock_period));
ccfifo_write(EMC_PMACRO_DATA_RX_TERM_MODE, current_timing->burst_regs[EMC_PMACRO_DATA_RX_TERM_MODE_INDEX], 0);
}
ccfifo_write(EMC_DBG, emc_dbg_o, 0);
if (opt_zcal_en_cc) {
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, (1000000 / source_clock_period));
if (dram_dev_num == TWO_RANK) {
if (shared_zq_resistor) {
if (!(train_ca || train_ca_vref) || train_swap_rank) {
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, (1000000 / source_clock_period));
if (!(train_ca || train_ca_vref))
ccfifo_write(EMC_MRW3, ((mr13_flip_fspop ^ 0xC0) & 0xF3FFFFF7) | 0xC000000, 0);
}
ccfifo_write(EMC_SELF_REF, 0x100, 0);
skip_zqcal = true;
} else {
if ((train_ca || train_ca_vref) && !train_swap_rank) {
ccfifo_write(EMC_SELF_REF, 0x100, 0);
skip_zqcal = true;
} else {
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, (1000000 / source_clock_period));
}
}
}
}
if (!skip_zqcal) {
if (!(train_ca || train_ca_vref))
ccfifo_write(EMC_MRW3, ((mr13_flip_fspop ^ 0xC0) & 0xF3FFFFF7) | 0xC000000, 0);
ccfifo_write(EMC_SELF_REF, 0x100, 0);
}
}
if (!skip_zqcal) {
/* Step 17:
* MANSR exit self refresh.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 17\n");
if ((opt_dvfs_mode == MAN_SR) && (dram_type != DRAM_TYPE_LPDDR4))
ccfifo_write(EMC_SELF_REF, 0, 0);
/* Step 18:
* Send MRWs to LPDDR3/DDR3.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 18\n");
if (dram_type == DRAM_TYPE_LPDDR2) {
ccfifo_write(EMC_MRW2, next_timing->emc_mrw2, 0);
ccfifo_write(EMC_MRW, next_timing->emc_mrw, 0);
if (is_lpddr3) {
ccfifo_write(EMC_MRW4, next_timing->emc_mrw4, 0);
}
} else if (dram_type == DRAM_TYPE_DDR3) {
if (opt_dll_mode == DLL_ON) {
ccfifo_write(EMC_EMRS, next_timing->emc_emrs & ~EMC_EMRS_USE_EMRS_LONG_CNT, 0);
}
ccfifo_write(EMC_EMRS2, next_timing->emc_emrs2 & ~EMC_EMRS2_USE_EMRS2_LONG_CNT, 0);
ccfifo_write(EMC_MRS, next_timing->emc_mrs | EMC_EMRS_USE_EMRS_LONG_CNT, 0);
}
/* Step 19:
* ZQCAL for LPDDR3/DDR3
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 19\n");
if (opt_zcal_en_cc) {
if (dram_type == DRAM_TYPE_LPDDR2) {
uint32_t r;
uint32_t zq_op = opt_cc_short_zcal ? 0x56 : 0xAB;
uint32_t zcal_wait_time_ps = opt_cc_short_zcal ? 90000 : 360000;
uint32_t zcal_wait_time_clocks = div_o3(zcal_wait_time_ps, destination_clock_period);
r = zcal_wait_time_clocks << EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT | zcal_wait_time_clocks << EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT;
ccfifo_write(EMC_MRS_WAIT_CNT2, r, 0);
ccfifo_write(EMC_MRW, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | EMC_MRW_USE_MRW_EXT_CNT | 10 << EMC_MRW_MRW_MA_SHIFT | zq_op << EMC_MRW_MRW_OP_SHIFT, 0);
if (dram_dev_num == TWO_RANK) {
r = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | EMC_MRW_USE_MRW_EXT_CNT | 10 << EMC_MRW_MRW_MA_SHIFT | zq_op << EMC_MRW_MRW_OP_SHIFT;
ccfifo_write(EMC_MRW, r, 0);
}
} else if (dram_type == DRAM_TYPE_DDR3) {
uint32_t zq_op = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG;
ccfifo_write(EMC_ZQ_CAL, zq_op | 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
if (dram_dev_num == TWO_RANK) {
ccfifo_write(EMC_ZQ_CAL, zq_op | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
}
}
}
}
if (bg_regulator_mode_change) {
emc_set_shadow_bypass(ACTIVE);
uint32_t bg_regulator_switch_complete_wait_clks = ramp_up_wait > 1250000 ? 0 : (1250000 - ramp_up_wait) / destination_clock_period;
if (dvfs_with_training) {
bg_regulator_switch_complete_wait_clks = (1250000 / source_clock_period);
ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0, current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], bg_regulator_switch_complete_wait_clks);
} else {
ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0, next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], bg_regulator_switch_complete_wait_clks);
}
emc_set_shadow_bypass(ASSEMBLY);
}
/* Step 20:
* Issue ref and optional QRST.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 20\n");
if (dvfs_with_training || (dram_type != DRAM_TYPE_LPDDR4))
ccfifo_write(EMC_REF, 0, 0);
if (opt_do_sw_qrst) {
ccfifo_write(EMC_ISSUE_QRST, 1, 0);
ccfifo_write(EMC_ISSUE_QRST, 0, 2);
}
/* Step 21:
* Restore ZCAL and ZCAL interval.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 21\n");
if (save_restore_clkstop_pd || opt_zcal_en_cc) {
emc_set_shadow_bypass(ACTIVE);
if (opt_zcal_en_cc) {
if (dvfs_with_training) {
ccfifo_write(EMC_ZCAL_INTERVAL, current_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX], 0);
} else if (dram_type != DRAM_TYPE_LPDDR4) {
ccfifo_write(EMC_ZCAL_INTERVAL, next_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX], 0);
}
}
if (save_restore_clkstop_pd) {
ccfifo_write(EMC_CFG, next_timing->burst_regs[EMC_CFG_INDEX] & ~EMC_CFG_DYN_SELF_REF, 0);
}
if (dvfs_with_training && (dram_type == DRAM_TYPE_LPDDR4)) {
ccfifo_write(EMC_SEL_DPD_CTRL, current_timing->emc_sel_dpd_ctrl, 0);
}
emc_set_shadow_bypass(ASSEMBLY);
}
/* Step 22:
* Restore EMC_CFG_PIPE_CLK.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 22\n");
ccfifo_write(EMC_CFG_PIPE_CLK, emc_cfg_pipe_clk_o, 0);
if (bg_regulator_mode_change) {
if (enable_bg_regulator) {
emc_write(next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, EMC_PMACRO_BG_BIAS_CTRL_0);
} else {
emc_write(next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, EMC_PMACRO_BG_BIAS_CTRL_0);
}
}
/* Step 23:
* Do clock change.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 23\n");
if (dvfs_with_training) {
car->clk_source_emc_safe = car->clk_source_emc;
change_dll_src(current_timing, car->clk_source_emc);
}
uint32_t cfg_dig_dll_tmp = emc_read(EMC_CFG_DIG_DLL);
cfg_dig_dll_tmp |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
cfg_dig_dll_tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
cfg_dig_dll_tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
cfg_dig_dll_tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
cfg_dig_dll_tmp = (cfg_dig_dll_tmp & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
emc_write(cfg_dig_dll_tmp, EMC_CFG_DIG_DLL);
car->clk_source_emc = next_clk_src;
wait_for_update(EMC_INTSTATUS, EMC_INTSTATUS_CLKCHANGE_COMPLETE, true, REG_EMC);
/* Step 24:
* Save training results.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 24\n");
if (dvfs_with_training) {
uint32_t emc_dbg_tmp = emc_read(EMC_DBG);
emc_write(emc_dbg_tmp | 1, EMC_DBG); /* Set READ_MUX to ASSEMBLY. */
/* Save CA results. */
if (train_ca) {
next_timing->trim_perch_regs[REG_EMC0_EMC_CMD_BRLSHFT_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_CMD_BRLSHFT_0);
next_timing->trim_perch_regs[REG_EMC1_EMC_CMD_BRLSHFT_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_CMD_BRLSHFT_1): 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1,EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5) : 0;
if (train_self_refresh) {
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2);
}
}
/* Save CA_VREF results. */
if (train_ca_vref) {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW10_INDEX] = (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) & 0xFFFF) | 0x880C0000;
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW10_INDEX] = (channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) & 0xFFFF : 0) | 0x880C0000;
if (dram_dev_num == TWO_RANK) {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW11_INDEX] = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) >> 16) & 0xFF) | (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) >> 24 << 8) | (0x480C0000 & 0xFFFFFF00);
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW11_INDEX] = (((channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) : 0) >> 16) & 0xFF) | ((channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) : 0) >> 24 << 8) | (0x480C0000 & 0xFFFFFF00);
} else {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW11_INDEX] = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) >> 16) & 0xFF) | (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) >> 24 << 8) | (0xC80C0000 & 0xFFFFFF00);
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW11_INDEX] = (((channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) : 0) >> 16) & 0xFF) | ((channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) : 0) >> 24 << 8) | (0xC80C0000 & 0xFFFFFF00);
}
}
/* Save QUSE results. */
if (train_quse || train_rd) {
next_timing->trim_perch_regs[REG_EMC0_EMC_QUSE_BRLSHFT_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_QUSE_BRLSHFT_0);
next_timing->trim_perch_regs[REG_EMC1_EMC_QUSE_BRLSHFT_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_QUSE_BRLSHFT_1) : 0;
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_QUSE_DDLL_RANK0_0);
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK0_1_INDEX]= emc_read_per_ch(REG_EMC0, EMC_PMACRO_QUSE_DDLL_RANK0_1);
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK0_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_QUSE_DDLL_RANK0_2) : 0;
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK0_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_QUSE_DDLL_RANK0_3) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_perch_regs[REG_EMC0_EMC_QUSE_BRLSHFT_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_QUSE_BRLSHFT_2);
next_timing->trim_perch_regs[REG_EMC1_EMC_QUSE_BRLSHFT_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_QUSE_BRLSHFT_3) : 0;
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_QUSE_DDLL_RANK1_0);
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_QUSE_DDLL_RANK1_1);
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK1_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_QUSE_DDLL_RANK1_2) : 0;
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK1_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_QUSE_DDLL_RANK1_3) : 0;
}
}
/* Save QUSE_VREF results. */
if (train_quse_vref) {
if (dram_dev_num == TWO_RANK) {
uint32_t emc0_opt_dqs_array[4] = {0};
uint32_t emc1_opt_dqs_array[4] = {0};
uint32_t emc1_training_opt_dqs_ib_vref_rank0_val = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_DQS_IB_VREF_RANK0) : 0;
uint32_t emc1_training_opt_dqs_ib_vref_rank1_val = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_DQS_IB_VREF_RANK1) : 0;
for (int i = 0; i < 4; i++) {
emc0_opt_dqs_array[i] = (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQS_IB_VREF_RANK0) >> (8 * i)) & 0xFF;
emc1_opt_dqs_array[i] = (emc1_training_opt_dqs_ib_vref_rank0_val >> (8 * i)) & 0xFF;
}
uint32_t ib_vref_dqs_0 = 0;
uint32_t ib_vref_dqs_1 = 0;
for (int i = 0; i < 4; i++)
{
ib_vref_dqs_0 |= (emc0_opt_dqs_array[i] + ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQS_IB_VREF_RANK1) >> (8 * i)) & 0xFF)) >> 1 << (8 * i);
ib_vref_dqs_1 |= (emc1_opt_dqs_array[i] + ((emc1_training_opt_dqs_ib_vref_rank1_val >> (8 * i)) & 0xFF)) >> 1 << (8 * i);
}
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQS_0_INDEX] = ib_vref_dqs_0;
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQS_1_INDEX] = ib_vref_dqs_1;
}
else
{
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQS_0_INDEX] = emc_read(EMC_PMACRO_IB_VREF_DQS_0);
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQS_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_VREF_DQS_1) : 0;
}
}
/* Save RD results. */
if (train_rd) {
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2_INDEX]= channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3) : 0;
}
if (train_self_refresh) {
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2) : 0;
}
}
/* Save RD_VREF results. */
if (train_rd_vref) {
uint8_t ib_vref_dq_byte0_icr = (emc_read(EMC_PMACRO_IB_VREF_DQ_0) & 0x7F) + (next_timing->save_restore_mod_regs[0] & 0x7F);
if (next_timing->save_restore_mod_regs[0] & 0x80000000)
ib_vref_dq_byte0_icr = (emc_read(EMC_PMACRO_IB_VREF_DQ_0) & 0x7F) - (next_timing->save_restore_mod_regs[0] & 0x7F);
uint8_t ib_vref_dq_byte1_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 8) & 0x7F) + (next_timing->save_restore_mod_regs[1] & 0x7F);
if (next_timing->save_restore_mod_regs[1] & 0x80000000)
ib_vref_dq_byte1_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 8) & 0x7F) - (next_timing->save_restore_mod_regs[1] & 0x7F);
uint8_t ib_vref_dq_byte2_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 16) & 0x7F) + (next_timing->save_restore_mod_regs[2] & 0x7F);
if (next_timing->save_restore_mod_regs[2] & 0x80000000)
ib_vref_dq_byte2_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 16) & 0x7F) - (next_timing->save_restore_mod_regs[2] & 0x7F);
uint8_t ib_vref_dq_byte3_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 24) & 0x7F) + (next_timing->save_restore_mod_regs[3] & 0x7F);
if (next_timing->save_restore_mod_regs[3] & 0x80000000)
ib_vref_dq_byte3_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 24) & 0x7F) - (next_timing->save_restore_mod_regs[3] & 0x7F);
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQ_0_INDEX] = ((ib_vref_dq_byte0_icr & 0x7F) | (ib_vref_dq_byte1_icr & 0x7F) << 8) | ((ib_vref_dq_byte2_icr & 0x7F) << 16) | ((ib_vref_dq_byte3_icr & 0x7F) << 24);
uint8_t ib_vref_dq_byte4_icr = (emc_read(EMC_PMACRO_IB_VREF_DQ_1) & 0x7F) + (next_timing->save_restore_mod_regs[4] & 0x7F);
if (next_timing->save_restore_mod_regs[4] & 0x80000000)
ib_vref_dq_byte4_icr = (emc_read(EMC_PMACRO_IB_VREF_DQ_1) & 0x7F) - (next_timing->save_restore_mod_regs[4] & 0x7F);
uint8_t ib_vref_dq_byte5_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 8) & 0x7F) + (next_timing->save_restore_mod_regs[5] & 0x7F);
if (next_timing->save_restore_mod_regs[5] & 0x80000000)
ib_vref_dq_byte5_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 8) & 0x7F) - (next_timing->save_restore_mod_regs[5] & 0x7F);
uint8_t ib_vref_dq_byte6_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 16) & 0x7F) + (next_timing->save_restore_mod_regs[6] & 0x7F);
if (next_timing->save_restore_mod_regs[6] & 0x80000000)
ib_vref_dq_byte6_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 16) & 0x7F) - (next_timing->save_restore_mod_regs[6] & 0x7F);
uint8_t ib_vref_dq_byte7_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 24) & 0x7F) + (next_timing->save_restore_mod_regs[7] & 0x7F);
if (next_timing->save_restore_mod_regs[7] & 0x80000000)
ib_vref_dq_byte7_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 24) & 0x7F) - (next_timing->save_restore_mod_regs[7] & 0x7F);
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQ_1_INDEX] = ((ib_vref_dq_byte4_icr & 0x7F) | (ib_vref_dq_byte5_icr & 0x7F) << 8) | ((ib_vref_dq_byte6_icr & 0x7F) << 16) | ((ib_vref_dq_byte7_icr & 0x7F) << 24);
}
}
/* Save WR results. */
if (train_wr) {
next_timing->trim_perch_regs[REG_EMC0_EMC_DATA_BRLSHFT_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_DATA_BRLSHFT_0);
next_timing->trim_perch_regs[REG_EMC1_EMC_DATA_BRLSHFT_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_DATA_BRLSHFT_0) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_perch_regs[REG_EMC0_EMC_DATA_BRLSHFT_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_DATA_BRLSHFT_1);
next_timing->trim_perch_regs[REG_EMC1_EMC_DATA_BRLSHFT_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_DATA_BRLSHFT_1) : 0;
}
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3) : 0;
}
if (train_self_refresh) {
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2) : 0;
}
}
/* Save WR_VREF results. */
if (train_wr_vref) {
uint32_t emc1_ranks_sub_partitions = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_DQ_OB_VREF) : 0;
uint8_t emc0_ib_vref_dq_byte8_modded_plus = next_timing->save_restore_mod_regs[8] + emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF);
if (next_timing->save_restore_mod_regs[8] & 0x80000000)
emc0_ib_vref_dq_byte8_modded_plus = emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) - next_timing->save_restore_mod_regs[8];
uint8_t emc0_mrw12_op_sp1 = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) & 0xFFFF) >> 8) + next_timing->save_restore_mod_regs[9];
if (next_timing->save_restore_mod_regs[9] & 0x80000000)
emc0_mrw12_op_sp1 = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) & 0xFFFF) >> 8) - next_timing->save_restore_mod_regs[9];
uint8_t emc0_mrw13_op_sp0 = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) >> 16) & 0xFF) + next_timing->save_restore_mod_regs[8];
if (next_timing->save_restore_mod_regs[8] & 0x80000000)
emc0_mrw13_op_sp0 = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) >> 16) & 0xFF) - next_timing->save_restore_mod_regs[8];
uint8_t emc0_ib_vref_dq_byte9_modded_a_plus = next_timing->save_restore_mod_regs[9] + (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) >> 24);
if (next_timing->save_restore_mod_regs[9] & 0x80000000)
emc0_ib_vref_dq_byte9_modded_a_plus = (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) >> 24) - (uint8_t)next_timing->save_restore_mod_regs[9];
uint8_t emc0_ib_vref_dq_byte10_modded_plus = emc1_ranks_sub_partitions + next_timing->save_restore_mod_regs[10];
if (next_timing->save_restore_mod_regs[10] & 0x80000000)
emc0_ib_vref_dq_byte10_modded_plus = emc1_ranks_sub_partitions - next_timing->save_restore_mod_regs[10];
uint8_t emc0_ib_vref_dq_byte11_modded_plus = ((emc1_ranks_sub_partitions & 0xFFFF) >> 8) + next_timing->save_restore_mod_regs[11];
if (next_timing->save_restore_mod_regs[11] & 0x80000000)
emc0_ib_vref_dq_byte11_modded_plus = ((emc1_ranks_sub_partitions & 0xFFFF) >> 8) - next_timing->save_restore_mod_regs[11];
uint8_t emc1_mrw13_op_sp0 = ((emc1_ranks_sub_partitions >> 16) & 0xFF) + next_timing->save_restore_mod_regs[10];
if (next_timing->save_restore_mod_regs[10] & 0x80000000)
emc1_mrw13_op_sp0 = ((emc1_ranks_sub_partitions >> 16) & 0xFF) - next_timing->save_restore_mod_regs[10];
uint8_t emc1_mrw13_op_sp1 = (emc1_ranks_sub_partitions >> 24) + next_timing->save_restore_mod_regs[11];
if (next_timing->save_restore_mod_regs[11] & 0x80000000)
emc1_mrw13_op_sp1 = (emc1_ranks_sub_partitions >> 24) - next_timing->save_restore_mod_regs[11];
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW12_INDEX] = (uint8_t)emc0_ib_vref_dq_byte10_modded_plus | 0x880E0000 | (emc0_ib_vref_dq_byte11_modded_plus << 8);
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW12_INDEX] = emc0_ib_vref_dq_byte8_modded_plus | 0x880E0000 | (emc0_mrw12_op_sp1 << 8);
if (dram_dev_num == TWO_RANK) {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW13_INDEX] = emc0_ib_vref_dq_byte9_modded_a_plus << 8 | emc0_mrw13_op_sp0 | 0x480E0000;
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW13_INDEX] = (emc1_mrw13_op_sp1 << 8) | emc1_mrw13_op_sp0 | 0x480E0000;
} else {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW13_INDEX] = emc0_ib_vref_dq_byte9_modded_a_plus << 8 | emc0_mrw13_op_sp0 | 0xC80E0000;
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW13_INDEX] = (emc1_mrw13_op_sp1 << 8) | emc1_mrw13_op_sp0 | 0xC80E0000;
}
}
}
emc_write(emc_dbg_tmp, EMC_DBG);
}
/* Step 25:
* Program MC updown registers.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 25\n");
if ((next_timing->rate > current_timing->rate) && !dvfs_with_training) {
for (int i = 0; i < next_timing->num_up_down; i++) {
mc_write(next_timing->la_scale_regs[i], la_scale_regs_off[i]);
}
/* Request a timing update. */
emc_timing_update(channel_mode);
}
/* Step 26:
* Restore ZCAL registers.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 26\n");
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], EMC_ZCAL_WAIT_CNT);
emc_write(next_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX], EMC_ZCAL_INTERVAL);
emc_set_shadow_bypass(ASSEMBLY);
}
if ((dram_type != DRAM_TYPE_LPDDR4)
&& opt_zcal_en_cc
&& !opt_short_zcal
&& opt_cc_short_zcal)
{
udelay(2);
emc_set_shadow_bypass(ACTIVE);
if (dram_type == DRAM_TYPE_LPDDR2) {
emc_write(next_timing->burst_regs[EMC_MRS_WAIT_CNT_INDEX], EMC_MRS_WAIT_CNT);
} else if (dram_type == DRAM_TYPE_DDR3) {
emc_write(next_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], EMC_ZCAL_WAIT_CNT);
}
emc_set_shadow_bypass(ASSEMBLY);
}
/* Step 27:
* Restore EMC_CFG, FDPD registers.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 27\n");
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_CFG_INDEX], EMC_CFG);
emc_set_shadow_bypass(ASSEMBLY);
emc_write(next_timing->emc_fdpd_ctrl_cmd_no_ramp, EMC_FDPD_CTRL_CMD_NO_RAMP);
emc_write(next_timing->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
/* Step 28:
* Training recover.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 28\n");
if (dvfs_with_training && (dram_type == DRAM_TYPE_LPDDR4)) {
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_CFG_INDEX], EMC_CFG);
emc_write(next_timing->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
emc_write(current_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], EMC_ZCAL_WAIT_CNT);
emc_write(current_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX], EMC_ZCAL_INTERVAL);
emc_write(current_timing->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2);
emc_write(current_timing->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3);
emc_write(current_timing->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4);
emc_write(current_timing->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5);
emc_write(current_timing->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6);
emc_write(current_timing->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7);
emc_write(current_timing->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8);
emc_set_shadow_bypass(ASSEMBLY);
emc_write(next_timing->burst_regs[EMC_TR_DVFS_INDEX] & ~EMC_TR_DVFS_TRAINING_DVFS, EMC_TR_DVFS);
}
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX], EMC_PMACRO_AUTOCAL_CFG_COMMON);
emc_set_shadow_bypass(ASSEMBLY);
/* Step 29:
* Power fix WAR.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 29\n");
emc_write(EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7,
EMC_PMACRO_CFG_PM_GLOBAL_0);
emc_write(EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR, EMC_PMACRO_TRAINING_CTRL_0);
emc_write(EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR, EMC_PMACRO_TRAINING_CTRL_1);
emc_write(0, EMC_PMACRO_CFG_PM_GLOBAL_0);
/* Step 30:
* Re-enable autocal.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 30\n");
if (dvfs_with_training) {
emc_auto_cal_config = current_timing->emc_auto_cal_config;
/* Restore FSP to account for switch back. Only needed in training. */
g_fsp_for_next_freq = !g_fsp_for_next_freq;
} else {
emc_auto_cal_config = next_timing->emc_auto_cal_config;
if (next_timing->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) {
dll_enable_stall(channel_mode);
}
}
emc_write(emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - done!\n");
}
static void do_periodic_emc_compensation(tegra_emc_timing_t* current_timing) {
uint32_t reg_count = 10;
uint32_t reg_list[] = {
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
EMC_DATA_BRLSHFT_0,
EMC_DATA_BRLSHFT_1
};
if (current_timing->periodic_training) {
int dram_dev_num = ((mc_read(MC_EMEM_ADR_CFG) & 1) + 1);
bool channel_mode = ((current_timing->burst_regs[EMC_FBIO_CFG7_INDEX] >> 2) & 1);
uint32_t emc_cfg_o = emc_read(EMC_CFG);
uint32_t emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | EMC_CFG_DRAM_CLKSTOP_PD | EMC_CFG_DRAM_CLKSTOP_PD);
/*
* 1. Power optimizations should be off.
*/
emc_write(emc_cfg, EMC_CFG);
/* Does emc_timing_update() for above changes. */
dll_disable(channel_mode);
if (dram_dev_num == TWO_RANK) {
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, false, REG_EMC1);
} else {
wait_for_update(EMC_EMC_STATUS, 0x10, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, 0x10, false, REG_EMC1);
}
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, false, REG_EMC1);
wait_for_update(EMC_EMC_STATUS, 0x01, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, 0x01, false, REG_EMC1);
uint32_t emc_cfg_update = emc_read(EMC_CFG_UPDATE);
emc_write((emc_cfg_update & ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK) | (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT), EMC_CFG_UPDATE);
/*
* 2. osc kick off - this assumes training and dvfs have set
* correct MR23.
*/
start_periodic_compensation();
/*
* 3. Let dram capture its clock tree delays.
*/
udelay((actual_osc_clocks(current_timing->run_clocks) * 1000) / current_timing->rate + 1);
/*
* 4. Check delta wrt previous values (save value if margin
* exceeds what is set in table).
*/
uint32_t del = periodic_compensation_handler(current_timing, current_timing, dram_dev_num, channel_mode, PERIODIC_TRAINING_SEQUENCE);
/*
* 5. Apply compensation w.r.t. trained values (if clock tree
* has drifted more than the set margin).
*/
if (current_timing->tree_margin < ((del * 128 * (current_timing->rate / 1000)) / 1000000)) {
for (int i = 0; i < reg_count; i++) {
uint32_t tmp = apply_periodic_compensation_trimmer(current_timing, reg_list[i]);
emc_write(tmp, reg_list[i]);
}
}
emc_write(emc_cfg_o, EMC_CFG);
/*
* 6. Timing update actually applies the new trimmers.
*/
emc_timing_update(channel_mode);
/* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */
emc_write(emc_cfg_update, EMC_CFG_UPDATE);
/* 6.2. Restore the DLL. */
dll_enable(channel_mode);
}
}
static void train_set_clock(tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, bool update_clk, uint32_t next_clk_src) {
/* Check for dual channel LPDDR4 */
bool dual_channel_lpddr4_case = ((emc_read(EMC_FBIO_CFG7) & EMC_FBIO_CFG7_CH0_ENABLE) & (emc_read(EMC_FBIO_CFG7) & EMC_FBIO_CFG7_CH1_ENABLE));
/* Get the DRAM type */
uint32_t dram_type = (next_timing->burst_regs[EMC_FBIO_CFG5_INDEX] & 0x03);
if (g_write_training_pattern) {
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Writing training patterns...\n");
/* Write the training data into pattern RAM */
for (int i = 0; i < 0x100; i++) {
uint32_t training_pattern_val = g_ram_pattern_dq[i + (next_timing->training_pattern * 0x100)];
/* Write the DQ data into pattern RAM */
emc_write(training_pattern_val, EMC_TRAINING_PATRAM_DQ);
training_pattern_val = g_ram_pattern_dmi[i + (next_timing->training_pattern * 0x100)];
/* Write the DMI data into pattern RAM */
emc_write(training_pattern_val & 0x0F, EMC_TRAINING_PATRAM_DMI);
/* Enable writing into pattern RAM and select the offset */
emc_write(0x80000000 + i, EMC_TRAINING_PATRAM_CTRL);
}
}
/* Only write the training pattern once */
g_write_training_pattern = false;
if (next_timing->needs_training && !next_timing->trained) {
uint32_t training_flags = next_timing->needs_training;
/* Read from MC_EMEM_ADR_CFG */
uint32_t dram_dev_num = mc_read(MC_EMEM_ADR_CFG);
int training_params_idx = 0;
int training_params[8] = {};
if (training_flags & 0x03) {
training_params_idx = 1;
training_params[0] = (training_flags & 0x203);
if (dram_dev_num & 0x01) {
training_params_idx = 2;
training_params[1] = (training_flags & 0x303);
}
}
if ((dram_dev_num & 0x01) && (training_flags & 0x0C)) {
training_params[training_params_idx] = (training_flags & 0x20C);
training_params[training_params_idx + 1] = (training_flags & 0x204);
training_params_idx += 2;
} else if (training_flags & 0x0C) {
training_params[training_params_idx++] = (training_flags & 0x20C);
}
if (training_flags & 0xF0)
training_params[training_params_idx++] = (training_flags & 0x2F0);
for (int i = 0; i < training_params_idx; i++) {
/* Adjust the clock */
set_clock(current_timing, next_timing, training_params[i], next_clk_src);
/* Change CFG_SWAP to ASSEMBLY_ONLY */
uint32_t emc_dbg = emc_read(EMC_DBG);
emc_dbg = ((emc_dbg & 0xF3FFFFFF) | 0x8000000);
emc_write(emc_dbg, EMC_DBG);
/* Change UPDATE_AUTO_CAL_IN_UPDATE to ALWAYS */
uint32_t emc_cfg_update = emc_read(EMC_CFG_UPDATE);
emc_cfg_update = ((emc_cfg_update & 0xFFFFFFF9) | 0x04);
emc_write(emc_cfg_update, EMC_CFG_UPDATE);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Change UPDATE_AUTO_CAL_IN_UPDATE to NEVER */
emc_cfg_update = emc_read(EMC_CFG_UPDATE);
emc_cfg_update &= 0xFFFFFFF9;
emc_write(emc_cfg_update, EMC_CFG_UPDATE);
/* Change CFG_SWAP to ACTIVE_ONLY */
emc_dbg = emc_read(EMC_DBG);
emc_dbg &= 0xF3FFFFFF;
emc_write(emc_dbg, EMC_DBG);
/* Disable DLL and change CFG_DLL_MODE to RUN_PERIODIC */
uint32_t emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll = ((emc_cfg_dig_dll & 0xFFFFFF3E) | 0x80);
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Disable or enable DLL */
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
if (next_timing->burst_regs[EMC_CFG_DIG_DLL_INDEX] == 0x01)
emc_cfg_dig_dll |= 0x01;
else
emc_cfg_dig_dll &= 0xFFFFFFFE;
/* Change CFG_DLL_MODE to RUN_PERIODIC */
emc_cfg_dig_dll = ((emc_cfg_dig_dll & 0xFFFFFF3F) | 0x80);
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait for DLL_LOCK to be set */
uint32_t emc_dig_dll_status = 0;
do {
emc_dig_dll_status = emc_read(EMC_DIG_DLL_STATUS);
} while (!(emc_dig_dll_status & EMC_DIG_DLL_STATUS_DLL_LOCK));
/* Check if DRAM is LPDDR4 */
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_write(current_timing->burst_regs[EMC_RP_INDEX], EMC_RP);
emc_write(current_timing->burst_regs[EMC_R2P_INDEX], EMC_R2P);
emc_write(current_timing->burst_regs[EMC_W2P_INDEX], EMC_W2P);
emc_write(current_timing->burst_regs[EMC_TRPAB_INDEX], EMC_TRPAB);
}
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
}
/* We've finished training */
next_timing->trained = 1;
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Memory is trained!\n");
}
/* Change the clock if requested */
if (update_clk)
set_clock(current_timing, next_timing, 0, next_clk_src);
}
static int train_one(int z_val, uint32_t next_rate, uint32_t current_rate, tegra_emc_timing_t* timing_tables, int timing_tables_count, TrainMode mode) {
int current_timing_table_idx = -1;
int next_timing_table_idx = -1;
tegra_emc_timing_t* current_timing_table;
tegra_emc_timing_t* next_timing_table;
uint32_t next_clk_src = 0;
/* Too many table entries */
if (timing_tables_count > 0x384)
return 4;
/* Locate the right tables for the transition */
for (int i = 0; i < timing_tables_count; i++) {
uint32_t rate = timing_tables[i].rate;
if (rate == current_rate)
current_timing_table_idx = i;
else if (rate == next_rate)
next_timing_table_idx = i;
}
/* Failed to find the tables. */
if ((current_timing_table_idx < 0) || (next_timing_table_idx < 0))
return 4;
current_timing_table = (tegra_emc_timing_t*)&timing_tables[current_timing_table_idx];
next_timing_table = (tegra_emc_timing_t*)&timing_tables[next_timing_table_idx];
uint32_t clk_src_emc_from = current_timing_table->clk_src_emc;
uint32_t clk_src_emc_to = next_timing_table->clk_src_emc;
uint32_t rate_from = current_timing_table->rate;
uint32_t rate_to = next_timing_table->rate;
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Changing rate from %d to %d!\n", rate_from, rate_to);
bool diff_freq = test_clk_ratio(rate_to, clk_src_emc_to, rate_from, clk_src_emc_from);
if (diff_freq) {
uint32_t emc_2x_clk_src = (clk_src_emc_from >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
if (emc_2x_clk_src & 0x03) {
if ((emc_2x_clk_src - TEGRA_EMC_SRC_PLLMB_UD) <= 1) {
g_is_pllmb = false;
}
} else {
g_is_pllmb = !g_is_pllmb;
}
/* Configure the PLL values */
next_clk_src = set_pll(rate_to, 0x9600, clk_src_emc_to, g_is_pllmb);
} else {
uint32_t emc_2x_clk_src = (clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
if ((emc_2x_clk_src != TEGRA_EMC_SRC_PLLMB) && emc_2x_clk_src) {
if (((emc_2x_clk_src - TEGRA_EMC_SRC_PLLM_UD) <= TEGRA_EMC_SRC_PLLC) && g_is_pllmb) {
next_clk_src = ((clk_src_emc_to & 0x1FFFFFFF) | (TEGRA_EMC_SRC_PLLMB_UD << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
}
} else if (g_is_pllmb) {
next_clk_src = ((clk_src_emc_to & 0x1FFFFFFF) | (TEGRA_EMC_SRC_PLLMB << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
} else {
next_clk_src = clk_src_emc_to;
}
}
if (mode == OP_SWITCH) {
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Train mode is OP_SWITCH!\n");
set_clock(current_timing_table, next_timing_table, false, next_clk_src);
g_active_timing_table_idx = next_timing_table_idx;
if (next_timing_table->periodic_training) {
do_periodic_emc_compensation(next_timing_table);
}
} else if (mode == OP_TRAIN) {
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Train mode is OP_TRAIN!\n");
train_set_clock(current_timing_table, next_timing_table, false, next_clk_src);
g_active_timing_table_idx = next_timing_table_idx;
if (diff_freq) {
g_is_pllmb = !g_is_pllmb;
}
} else if (mode == OP_TRAIN_SWITCH) {
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Train mode is OP_TRAIN_SWITCH!\n");
train_set_clock(current_timing_table, next_timing_table, true, next_clk_src);
g_active_timing_table_idx = next_timing_table_idx;
if (next_timing_table->periodic_training) {
do_periodic_emc_compensation(next_timing_table);
}
} else
return 4;
return 0;
}
Move memory training code into it's own stage (fusee-mtc)
2019-07-26 19:38:15 +00:00
void train_dram(void) {
fusee: Implement DRAM training: - Based on reverse engineered code and Peter De Schrijver's patches; - Complemented with CTCaer's minerva_tc project.
2018-11-12 01:55:16 +00:00
volatile tegra_car_t *car = car_get_regs();
tegra_emc_timing_t *timing_tables;
uint32_t dram_id = fuse_get_dram_id();
/* Find the right timing table set. */
if (dram_id == 0x01)
timing_tables = (tegra_emc_timing_t *)nx_abca2_dram_1;
else if ((dram_id == 0x00) || (dram_id == 0x02) || (dram_id == 0x03) || (dram_id == 0x04))
timing_tables = (tegra_emc_timing_t *)nx_abca2_dram_0234;
else
fatal_error("[MTC]: Missing tables for DRAM id %d!\n", dram_id);
/* Locate the right timing table. */
int boot_index = 0;
for (boot_index = 0; boot_index < MTC_TABLES_MAX_ENTRIES; boot_index++) {
print(SCREEN_LOG_LEVEL_DEBUG, "%d (%d kHz): %s\n", boot_index, timing_tables[boot_index].rate, timing_tables[boot_index].dvfs_ver);
if (car->clk_source_emc == timing_tables[boot_index].clk_src_emc)
break;
}
if (boot_index >= MTC_TABLES_MAX_ENTRIES) {
fatal_error("[MTC]: Failed to find timing table!\n");
}
/* Switch to 800Mhz. */
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Switching from %dMhz to 800Mhz\n", timing_tables[boot_index].rate / 1000);
train_one(0, 800000, timing_tables[boot_index].rate, timing_tables, MTC_TABLES_MAX_ENTRIES, OP_TRAIN_SWITCH);
/* Switch to 1600Mhz. */
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Switching from %dMhz to 1600Mhz\n", timing_tables[g_active_timing_table_idx].rate / 1000);
train_one(0, 1600000, timing_tables[g_active_timing_table_idx].rate, timing_tables, MTC_TABLES_MAX_ENTRIES, OP_TRAIN_SWITCH);
/* Wait a while. */
mdelay(100);
/* Do periodic compensation. */
do_periodic_emc_compensation((tegra_emc_timing_t*)&timing_tables[g_active_timing_table_idx]);
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Done!\n");
}
Reference in a new issue Copy permalink