random: use TinyMT instead of XorShift

This commit is contained in:
Michael Scire 2019-12-09 23:50:47 -08:00
parent 206b1a1b57
commit 1556a92a38
12 changed files with 415 additions and 175 deletions

View file

@ -44,7 +44,6 @@
#include "stratosphere/patcher.hpp"
#include "stratosphere/pm.hpp"
#include "stratosphere/reg.hpp"
#include "stratosphere/rnd.hpp"
#include "stratosphere/ro.hpp"
#include "stratosphere/settings.hpp"
#include "stratosphere/sf.hpp"

View file

@ -20,6 +20,7 @@
#include "os/os_memory_common.hpp"
#include "os/os_managed_handle.hpp"
#include "os/os_process_handle.hpp"
#include "os/os_random.hpp"
#include "os/os_mutex.hpp"
#include "os/os_condvar.hpp"
#include "os/os_rw_lock.hpp"

View file

@ -15,12 +15,13 @@
*/
#pragma once
#include <vapours.hpp>
#include "os_common_types.hpp"
namespace ams::rnd {
namespace ams::os {
/* Random utilities. */
void GenerateRandomBytes(void* out, size_t size);
void GenerateRandomBytes(void *dst, size_t size);
/* Convenience API. */
u32 GenerateRandomU32(u32 max = std::numeric_limits<u32>::max());
u64 GenerateRandomU64(u64 max = std::numeric_limits<u64>::max());

View file

@ -18,3 +18,4 @@
#include "util/util_compression.hpp"
#include "util/util_ini.hpp"
#include "util/util_tinymt.hpp"

View file

@ -0,0 +1,99 @@
/*
* Copyright (c) 2018-2019 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <vapours.hpp>
namespace ams::util {
/* Implementation of TinyMT (mersenne twister RNG). */
/* Like Nintendo, we will use the sample parameters. */
class TinyMT {
public:
static constexpr size_t NumStateWords = 4;
struct State {
u32 data[NumStateWords];
};
private:
static constexpr u32 ParamMat1 = 0x8F7011EE;
static constexpr u32 ParamMat2 = 0xFC78FF1F;
static constexpr u32 ParamTmat = 0x3793FDFF;
static constexpr u32 ParamMult = 0x6C078965;
static constexpr u32 ParamPlus = 0x0019660D;
static constexpr u32 ParamXor = 0x5D588B65;
static constexpr u32 TopBitmask = 0x7FFFFFFF;
static constexpr int MinimumInitIterations = 8;
static constexpr int NumDiscardedInitOutputs = 8;
private:
State state;
private:
/* Internal API. */
void FinalizeInitialization();
u32 GenerateRandomU24() { return (this->GenerateRandomU32() >> 8); }
static void GenerateInitialValuePlus(TinyMT::State *state, int index, u32 value);
static void GenerateInitialValueXor(TinyMT::State *state, int index);
public:
/* Public API. */
/* Initialization. */
void Initialize(u32 seed);
void Initialize(const u32 *seed, int seed_count);
/* State management. */
void GetState(TinyMT::State *out) const;
void SetState(const TinyMT::State *state);
/* Random generation. */
void GenerateRandomBytes(void *dst, size_t size);
u32 GenerateRandomU32();
inline u64 GenerateRandomU64() {
const u32 lo = this->GenerateRandomU32();
const u32 hi = this->GenerateRandomU32();
return (static_cast<u64>(hi) << 32) | static_cast<u64>(lo);
}
inline float GenerateRandomF32() {
/* Floats have 24 bits of mantissa. */
constexpr int MantissaBits = 24;
return GenerateRandomU24() * (1.0f / (1ul << MantissaBits));
}
inline double GenerateRandomF64() {
/* Doubles have 53 bits of mantissa. */
/* The smart way to generate 53 bits of random would be to use 32 bits */
/* from the first rnd32() call, and then 21 from the second. */
/* Nintendo does not. They use (32 - 5) = 27 bits from the first rnd32() */
/* call, and (32 - 6) bits from the second. We'll do what they do, but */
/* There's not a clear reason why. */
constexpr int MantissaBits = 53;
constexpr int Shift1st = (64 - MantissaBits) / 2;
constexpr int Shift2nd = (64 - MantissaBits) - Shift1st;
const u32 first = (this->GenerateRandomU32() >> Shift1st);
const u32 second = (this->GenerateRandomU32() >> Shift2nd);
return (1.0 * first * (1ul << (32 - Shift2nd)) + second) * (1.0 / (1ul << MantissaBits));
}
};
}

View file

@ -43,9 +43,9 @@ namespace ams::map {
}
const uintptr_t mem_end = mem_info.addr + mem_info.size;
if (mem_info.type == MemType_Reserved || mem_end < cur_base || (mem_end >> 31)) {
return svc::ResultOutOfMemory();
}
R_UNLESS(mem_info.type != MemType_Reserved, svc::ResultOutOfMemory());
R_UNLESS(cur_base <= mem_end, svc::ResultOutOfMemory());
R_UNLESS(mem_end <= static_cast<uintptr_t>(std::numeric_limits<s32>::max()), svc::ResultOutOfMemory());
cur_base = mem_end;
} while (true);
@ -61,22 +61,16 @@ namespace ams::map {
cur_base = address_space.aslr_base;
cur_end = cur_base + size;
if (cur_end <= cur_base) {
return svc::ResultOutOfMemory();
}
R_UNLESS(cur_base < cur_end, svc::ResultOutOfMemory());
while (true) {
if (address_space.heap_size && (address_space.heap_base <= cur_end - 1 && cur_base <= address_space.heap_end - 1)) {
/* If we overlap the heap region, go to the end of the heap region. */
if (cur_base == address_space.heap_end) {
return svc::ResultOutOfMemory();
}
R_UNLESS(cur_base != address_space.heap_end, svc::ResultOutOfMemory());
cur_base = address_space.heap_end;
} else if (address_space.alias_size && (address_space.alias_base <= cur_end - 1 && cur_base <= address_space.alias_end - 1)) {
/* If we overlap the alias region, go to the end of the alias region. */
if (cur_base == address_space.alias_end) {
return svc::ResultOutOfMemory();
}
R_UNLESS(cur_base != address_space.alias_end, svc::ResultOutOfMemory());
cur_base = address_space.alias_end;
} else {
R_ASSERT(svcQueryMemory(&mem_info, &page_info, cur_base));
@ -84,18 +78,13 @@ namespace ams::map {
*out_address = cur_base;
return ResultSuccess();
}
if (mem_info.addr + mem_info.size <= cur_base) {
return svc::ResultOutOfMemory();
}
R_UNLESS(cur_base < mem_info.addr + mem_info.size, svc::ResultOutOfMemory());
cur_base = mem_info.addr + mem_info.size;
if (cur_base >= address_space.aslr_end) {
return svc::ResultOutOfMemory();
}
R_UNLESS(cur_base < address_space.aslr_end, svc::ResultOutOfMemory());
}
cur_end = cur_base + size;
if (cur_base + size <= cur_base) {
return svc::ResultOutOfMemory();
}
R_UNLESS(cur_base < cur_base + size, svc::ResultOutOfMemory());
}
}
@ -103,19 +92,15 @@ namespace ams::map {
AddressSpaceInfo address_space;
R_TRY(GetProcessAddressSpaceInfo(&address_space, process_handle));
if (size > address_space.aslr_size) {
return ro::ResultOutOfAddressSpace();
}
R_UNLESS(size <= address_space.aslr_size, ro::ResultOutOfAddressSpace());
uintptr_t try_address;
for (unsigned int i = 0; i < LocateRetryCount; i++) {
try_address = address_space.aslr_base + (rnd::GenerateRandomU64(static_cast<u64>(address_space.aslr_size - size) >> 12) << 12);
try_address = address_space.aslr_base + (os::GenerateRandomU64(static_cast<u64>(address_space.aslr_size - size) / os::MemoryPageSize) * os::MemoryPageSize);
MappedCodeMemory tmp_mcm(process_handle, try_address, base_address, size);
R_TRY_CATCH(tmp_mcm.GetResult()) {
R_CATCH(svc::ResultInvalidCurrentMemoryState) {
continue;
}
R_CATCH(svc::ResultInvalidCurrentMemoryState) { continue; }
} R_END_TRY_CATCH;
if (!CanAddGuardRegionsInProcess(process_handle, try_address, size)) {
@ -134,14 +119,12 @@ namespace ams::map {
AddressSpaceInfo address_space;
R_TRY(GetProcessAddressSpaceInfo(&address_space, process_handle));
if (size > address_space.aslr_size) {
return ro::ResultOutOfAddressSpace();
}
R_UNLESS(size <= address_space.aslr_size, ro::ResultOutOfAddressSpace());
uintptr_t try_address;
for (unsigned int i = 0; i < LocateRetryCount; i++) {
while (true) {
try_address = address_space.aslr_base + (rnd::GenerateRandomU64(static_cast<u64>(address_space.aslr_size - size) >> 12) << 12);
try_address = address_space.aslr_base + (os::GenerateRandomU64(static_cast<u64>(address_space.aslr_size - size) / os::MemoryPageSize) * os::MemoryPageSize);
if (address_space.heap_size && (address_space.heap_base <= try_address + size - 1 && try_address <= address_space.heap_end - 1)) {
continue;
}
@ -153,9 +136,7 @@ namespace ams::map {
MappedCodeMemory tmp_mcm(process_handle, try_address, base_address, size);
R_TRY_CATCH(tmp_mcm.GetResult()) {
R_CATCH(svc::ResultInvalidCurrentMemoryState) {
continue;
}
R_CATCH(svc::ResultInvalidCurrentMemoryState) { continue; }
} R_END_TRY_CATCH;
if (!CanAddGuardRegionsInProcess(process_handle, try_address, size)) {

View file

@ -13,7 +13,11 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <stratosphere.hpp>
#include "rnd/rnd_api.hpp"
namespace ams::os::impl {
void InitializeRandomImpl(util::TinyMT *mt);
}

View file

@ -0,0 +1,34 @@
/*
* Copyright (c) 2018-2019 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stratosphere.hpp>
namespace ams::os::impl {
void InitializeRandomImpl(util::TinyMT *mt) {
/* Retrieve entropy from kernel. */
u32 seed[4];
static_assert(util::size(seed) == util::TinyMT::NumStateWords);
/* Nintendo does not check the result of these invocations, but we will for safety. */
/* Nintendo uses entropy values 0, 1 to seed the public TinyMT random, and values */
/* 2, 3 to seed os::detail::RngManager's private TinyMT random. */
R_ASSERT(svcGetInfo(reinterpret_cast<u64 *>(&seed[0]), InfoType_RandomEntropy, INVALID_HANDLE, 0));
R_ASSERT(svcGetInfo(reinterpret_cast<u64 *>(&seed[2]), InfoType_RandomEntropy, INVALID_HANDLE, 1));
mt->Initialize(seed, util::size(seed));
}
}

View file

@ -0,0 +1,61 @@
/*
* Copyright (c) 2018-2019 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stratosphere.hpp>
#include "impl/os_random_impl.hpp"
namespace ams::os {
namespace {
util::TinyMT g_random;
os::Mutex g_random_mutex;
bool g_initialized_random;
template<typename T>
inline T GenerateRandomTImpl(T max) {
static_assert(std::is_integral<T>::value && std::is_unsigned<T>::value);
const T EffectiveMax = (std::numeric_limits<T>::max() / max) * max;
T cur_rnd;
while (true) {
os::GenerateRandomBytes(&cur_rnd, sizeof(T));
if (cur_rnd < EffectiveMax) {
return cur_rnd % max;
}
}
}
}
void GenerateRandomBytes(void *dst, size_t size) {
std::scoped_lock lk(g_random_mutex);
if (!g_initialized_random) {
impl::InitializeRandomImpl(&g_random);
g_initialized_random = true;
}
g_random.GenerateRandomBytes(dst, size);
}
u32 GenerateRandomU32(u32 max) {
return GenerateRandomTImpl<u32>(max);
}
u64 GenerateRandomU64(u64 max) {
return GenerateRandomTImpl<u64>(max);
}
}

View file

@ -1,129 +0,0 @@
/*
* Copyright (c) 2018-2019 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <random>
#include <stratosphere.hpp>
namespace ams::rnd {
namespace {
/* Generator type. */
/* Official HOS uses TinyMT. This is high effort. Let's just use XorShift. */
/* https://en.wikipedia.org/wiki/Xorshift */
class XorShiftGenerator {
public:
using ResultType = uint32_t;
using result_type = ResultType;
static constexpr ResultType (min)() { return std::numeric_limits<ResultType>::min(); }
static constexpr ResultType (max)() { return std::numeric_limits<ResultType>::max(); }
static constexpr size_t SeedSize = 4;
private:
ResultType random_state[SeedSize];
public:
explicit XorShiftGenerator() {
/* Seed using process entropy. */
u64 val = 0;
for (size_t i = 0; i < SeedSize; i++) {
R_ASSERT(svcGetInfo(&val, InfoType_RandomEntropy, INVALID_HANDLE, i));
this->random_state[i] = ResultType(val);
}
}
explicit XorShiftGenerator(std::random_device &rd) {
for (size_t i = 0; i < SeedSize; i++) {
this->random_state[i] = ResultType(rd());
}
}
ResultType operator()() {
ResultType s, t = this->random_state[3];
t ^= t << 11;
t ^= t >> 8;
this->random_state[3] = this->random_state[2]; this->random_state[2] = this->random_state[1]; this->random_state[1] = (s = this->random_state[0]);
t ^= s;
t ^= s >> 19;
this->random_state[0] = t;
return t;
}
void discard(size_t n) {
for (size_t i = 0; i < n; i++) {
operator()();
}
}
};
/* Generator global. */
XorShiftGenerator g_rnd_generator;
/* Templated helpers. */
template<typename T>
T GenerateRandom(T max = std::numeric_limits<T>::max()) {
std::uniform_int_distribution<T> rnd(std::numeric_limits<T>::min(), max);
return rnd(g_rnd_generator);
}
}
void GenerateRandomBytes(void* _out, size_t size) {
uintptr_t out = reinterpret_cast<uintptr_t>(_out);
uintptr_t end = out + size;
/* Force alignment. */
if (out % sizeof(u16) && out < end) {
*reinterpret_cast<u8 *>(out) = GenerateRandom<u8>();
out += sizeof(u8);
}
if (out % sizeof(u32) && out < end) {
*reinterpret_cast<u16 *>(out) = GenerateRandom<u16>();
out += sizeof(u16);
}
if (out % sizeof(u64) && out < end) {
*reinterpret_cast<u32 *>(out) = GenerateRandom<u32>();
out += sizeof(u32);
}
/* Perform as many aligned writes as possible. */
while (out + sizeof(u64) <= end) {
*reinterpret_cast<u64 *>(out) = GenerateRandom<u64>();
out += sizeof(u64);
}
/* Do remainder writes. */
if (out + sizeof(u32) <= end) {
*reinterpret_cast<u32 *>(out) = GenerateRandom<u32>();
out += sizeof(u32);
}
if (out + sizeof(u16) <= end) {
*reinterpret_cast<u16 *>(out) = GenerateRandom<u16>();
out += sizeof(u16);
}
if (out + sizeof(u8) <= end) {
*reinterpret_cast<u8 *>(out) = GenerateRandom<u8>();
out += sizeof(u8);
}
}
u32 GenerateRandomU32(u32 max) {
return GenerateRandom<u32>(max);
}
u64 GenerateRandomU64(u64 max) {
return GenerateRandom<u64>(max);
}
}

View file

@ -0,0 +1,187 @@
/*
* Copyright (c) 2018-2019 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stratosphere.hpp>
namespace ams::util {
namespace {
constexpr inline u32 XorByShifted27(u32 value) {
return value ^ (value >> 27);
}
constexpr inline u32 XorByShifted30(u32 value) {
return value ^ (value >> 30);
}
}
void TinyMT::GenerateInitialValuePlus(TinyMT::State *state, int index, u32 value) {
u32 &state0 = state->data[(index + 0) % NumStateWords];
u32 &state1 = state->data[(index + 1) % NumStateWords];
u32 &state2 = state->data[(index + 2) % NumStateWords];
u32 &state3 = state->data[(index + 3) % NumStateWords];
const u32 x = XorByShifted27(state0 ^ state1 ^ state3) * ParamPlus;
const u32 y = x + index + value;
state0 = y;
state1 += x;
state2 += y;
}
void TinyMT::GenerateInitialValueXor(TinyMT::State *state, int index) {
u32 &state0 = state->data[(index + 0) % NumStateWords];
u32 &state1 = state->data[(index + 1) % NumStateWords];
u32 &state2 = state->data[(index + 2) % NumStateWords];
u32 &state3 = state->data[(index + 3) % NumStateWords];
const u32 x = XorByShifted27(state0 + state1 + state3) * ParamXor;
const u32 y = x - index;
state0 = y;
state1 ^= x;
state2 ^= y;
}
void TinyMT::Initialize(u32 seed) {
this->state.data[0] = seed;
this->state.data[1] = ParamMat1;
this->state.data[2] = ParamMat2;
this->state.data[3] = ParamTmat;
for (int i = 1; i < MinimumInitIterations; i++) {
const u32 mixed = XorByShifted30(this->state.data[(i - 1) % NumStateWords]);
this->state.data[i % NumStateWords] ^= mixed * ParamMult + i;
}
this->FinalizeInitialization();
}
void TinyMT::Initialize(const u32 *seed, int seed_count) {
this->state.data[0] = 0;
this->state.data[1] = ParamMat1;
this->state.data[2] = ParamMat2;
this->state.data[3] = ParamTmat;
{
const int num_init_iterations = std::max(seed_count, MinimumInitIterations);
GenerateInitialValuePlus(&this->state, 0, seed_count);
for (int i = 0; i < num_init_iterations; i++) {
GenerateInitialValuePlus(&this->state, (i + 1) % NumStateWords, (i < seed_count) ? seed[i] : 0);
}
for (int i = 0; i < static_cast<int>(NumStateWords); i++) {
GenerateInitialValueXor(&this->state, (i + 1 + num_init_iterations) % NumStateWords);
}
}
this->FinalizeInitialization();
}
void TinyMT::FinalizeInitialization() {
const u32 state0 = this->state.data[0] & TopBitmask;
const u32 state1 = this->state.data[1];
const u32 state2 = this->state.data[2];
const u32 state3 = this->state.data[3];
if (state0 == 0 && state1 == 0 && state2 == 0 && state3 == 0) {
this->state.data[0] = 'T';
this->state.data[1] = 'I';
this->state.data[2] = 'N';
this->state.data[3] = 'Y';
}
for (int i = 0; i < NumDiscardedInitOutputs; i++) {
this->GenerateRandomU32();
}
}
void TinyMT::GetState(TinyMT::State *out) const {
std::memcpy(out->data, this->state.data, sizeof(this->state));
}
void TinyMT::SetState(const TinyMT::State *state) {
std::memcpy(this->state.data, state->data, sizeof(this->state));
}
void TinyMT::GenerateRandomBytes(void *dst, size_t size) {
const uintptr_t start = reinterpret_cast<uintptr_t>(dst);
const uintptr_t end = start + size;
const uintptr_t aligned_start = util::AlignUp(start, 4);
const uintptr_t aligned_end = util::AlignDown(end, 4);
/* Make sure we're aligned. */
if (start < aligned_start) {
const u32 rnd = this->GenerateRandomU32();
std::memcpy(dst, &rnd, aligned_start - start);
}
/* Write as many aligned u32s as we can. */
{
u32 * cur_dst = reinterpret_cast<u32 *>(aligned_start);
u32 * const end_dst = reinterpret_cast<u32 *>(aligned_end);
while (cur_dst < end_dst) {
*(cur_dst++) = this->GenerateRandomU32();
}
}
/* Handle any leftover unaligned data. */
if (aligned_end < end) {
const u32 rnd = this->GenerateRandomU32();
std::memcpy(reinterpret_cast<void *>(aligned_end), &rnd, end - aligned_end);
}
}
u32 TinyMT::GenerateRandomU32() {
/* Advance state. */
const u32 x0 = (this->state.data[0] & TopBitmask) ^ this->state.data[1] ^ this->state.data[2];
const u32 y0 = this->state.data[3];
const u32 x1 = x0 ^ (x0 << 1);
const u32 y1 = y0 ^ (y0 >> 1) ^ x1;
const u32 state0 = this->state.data[1];
u32 state1 = this->state.data[2];
u32 state2 = x1 ^ (y1 << 10);
const u32 state3 = y1;
if ((y1 & 1) != 0) {
state1 ^= ParamMat1;
state2 ^= ParamMat2;
}
this->state.data[0] = state0;
this->state.data[1] = state1;
this->state.data[2] = state2;
this->state.data[3] = state3;
/* Temper. */
const u32 t1 = state0 + (state2 >> 8);
u32 t0 = state3 ^ t1;
if ((t1 & 1) != 0) {
t0 ^= ParamTmat;
}
return t0;
}
}

View file

@ -499,9 +499,10 @@ namespace ams::ldr {
/* Set Create Process output. */
uintptr_t aslr_slide = 0;
uintptr_t unused_size = (aslr_size - total_size);
uintptr_t free_size = (aslr_size - total_size);
if (out_cpi->flags & svc::CreateProcessFlag_EnableAslr) {
aslr_slide = ams::rnd::GenerateRandomU64(unused_size / os::MemoryBlockUnitSize) * os::MemoryBlockUnitSize;
/* Nintendo uses MT19937 (not os::GenerateRandomBytes), but we'll just use TinyMT for now. */
aslr_slide = os::GenerateRandomU64(free_size / os::MemoryBlockUnitSize) * os::MemoryBlockUnitSize;
}
/* Set out. */