emu-mirror/Ryujinx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Validate CPU virtual addresses on access (#1987)

Browse Source 
* Enable PTE null checks again

* Do address validation on EmitPtPointerLoad, and make it branchless

* PTC version increment

* Mask of pointer tag for exclusive access

* Move mask to the correct place

Co-authored-by: LDj3SNuD <35856442+LDj3SNuD@users.noreply.github.com>
This commit is contained in:
gdkchan 2021-02-16 15:04:19 -03:00 committed by GitHub
parent 6f1d964801
commit 715b605e95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 52 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
ARMeilleure/Instructions/InstEmitMemoryExHelper.cs
View File

@ -19,19 +19,8 @@ namespace ARMeilleure.Instructions
if (size == 4) if (size == 4)
{ {
Operand isUnalignedAddr = InstEmitMemoryHelper.EmitAddressCheck(context, address, size);
Operand lblFastPath = Label();
context.BranchIfFalse(lblFastPath, isUnalignedAddr);
// The call is not expected to return (it should throw).
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
context.MarkLabel(lblFastPath);
// Only 128-bit CAS is guaranteed to have a atomic load. // Only 128-bit CAS is guaranteed to have a atomic load.
Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: false); Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: false, 4);
Operand zero = context.VectorZero(); Operand zero = context.VectorZero();
@ -119,20 +108,8 @@ namespace ARMeilleure.Instructions
context.BranchIfTrue(lblExit, exFailed); context.BranchIfTrue(lblExit, exFailed);
// STEP 2: We have exclusive access, make sure that the address is valid. // STEP 2: We have exclusive access and the address is valid, attempt the store using CAS.
Operand isUnalignedAddr = InstEmitMemoryHelper.EmitAddressCheck(context, address, size); Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: true, size);
Operand lblFastPath = Label();
context.BranchIfFalse(lblFastPath, isUnalignedAddr);
// The call is not expected to return (it should throw).
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
// STEP 3: We have exclusive access and the address is valid, attempt the store using CAS.
context.MarkLabel(lblFastPath);
Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: true);
Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset())); Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
Operand exValue = size switch Operand exValue = size switch
@ -151,7 +128,7 @@ namespace ARMeilleure.Instructions
_ => context.CompareAndSwap(physAddr, exValue, value) _ => context.CompareAndSwap(physAddr, exValue, value)
}; };
// STEP 4: Check if we succeeded by comparing expected and in-memory values. // STEP 3: Check if we succeeded by comparing expected and in-memory values.
Operand storeFailed; Operand storeFailed;
if (size == 4) if (size == 4)

118
ARMeilleure/Instructions/InstEmitMemoryHelper.cs
View File

@ -127,11 +127,7 @@ namespace ARMeilleure.Instructions
Operand lblSlowPath = Label(); Operand lblSlowPath = Label();
Operand lblEnd = Label(); Operand lblEnd = Label();
Operand isUnalignedAddr = EmitAddressCheck(context, address, size); Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false, size);
context.BranchIfTrue(lblSlowPath, isUnalignedAddr);
Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false);
Operand value = null; Operand value = null;
@ -161,18 +157,7 @@ namespace ARMeilleure.Instructions
throw new ArgumentOutOfRangeException(nameof(size)); throw new ArgumentOutOfRangeException(nameof(size));
} }
Operand isUnalignedAddr = EmitAddressCheck(context, address, size); Operand physAddr = EmitPtPointerLoad(context, address, null, write: false, size);
Operand lblFastPath = Label();
context.BranchIfFalse(lblFastPath, isUnalignedAddr, BasicBlockFrequency.Cold);
// The call is not expected to return (it should throw).
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
context.MarkLabel(lblFastPath);
Operand physAddr = EmitPtPointerLoad(context, address, null, write: false);
return size switch return size switch
{ {
@ -195,11 +180,7 @@ namespace ARMeilleure.Instructions
Operand lblSlowPath = Label(); Operand lblSlowPath = Label();
Operand lblEnd = Label(); Operand lblEnd = Label();
Operand isUnalignedAddr = EmitAddressCheck(context, address, size); Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false, size);
context.BranchIfTrue(lblSlowPath, isUnalignedAddr);
Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false);
Operand value = null; Operand value = null;
@ -233,11 +214,7 @@ namespace ARMeilleure.Instructions
Operand lblSlowPath = Label(); Operand lblSlowPath = Label();
Operand lblEnd = Label(); Operand lblEnd = Label();
Operand isUnalignedAddr = EmitAddressCheck(context, address, size); Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: true, size);
context.BranchIfTrue(lblSlowPath, isUnalignedAddr);
Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: true);
Operand value = GetInt(context, rt); Operand value = GetInt(context, rt);
@ -270,18 +247,7 @@ namespace ARMeilleure.Instructions
throw new ArgumentOutOfRangeException(nameof(size)); throw new ArgumentOutOfRangeException(nameof(size));
} }
Operand isUnalignedAddr = EmitAddressCheck(context, address, size); Operand physAddr = EmitPtPointerLoad(context, address, null, write: true, size);
Operand lblFastPath = Label();
context.BranchIfFalse(lblFastPath, isUnalignedAddr, BasicBlockFrequency.Cold);
// The call is not expected to return (it should throw).
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
context.MarkLabel(lblFastPath);
Operand physAddr = EmitPtPointerLoad(context, address, null, write: true);
if (size < 3 && value.Type == OperandType.I64) if (size < 3 && value.Type == OperandType.I64)
{ {
@ -312,11 +278,7 @@ namespace ARMeilleure.Instructions
Operand lblSlowPath = Label(); Operand lblSlowPath = Label();
Operand lblEnd = Label(); Operand lblEnd = Label();
Operand isUnalignedAddr = EmitAddressCheck(context, address, size); Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: true, size);
context.BranchIfTrue(lblSlowPath, isUnalignedAddr);
Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: true);
Operand value = GetVec(rt); Operand value = GetVec(rt);
@ -338,61 +300,49 @@ namespace ARMeilleure.Instructions
context.MarkLabel(lblEnd); context.MarkLabel(lblEnd);
} }
public static Operand EmitAddressCheck(ArmEmitterContext context, Operand address, int size) public static Operand EmitPtPointerLoad(ArmEmitterContext context, Operand address, Operand lblSlowPath, bool write, int size)
{ {
ulong addressCheckMask = ~((1UL << context.Memory.AddressSpaceBits) - 1); int ptLevelBits = context.Memory.AddressSpaceBits - PageBits;
addressCheckMask |= (1u << size) - 1;
return context.BitwiseAnd(address, Const(address.Type, (long)addressCheckMask));
}
public static Operand EmitPtPointerLoad(ArmEmitterContext context, Operand address, Operand lblSlowPath, bool write)
{
int ptLevelBits = context.Memory.AddressSpaceBits - 12; // 12 = Number of page bits.
int ptLevelSize = 1 << ptLevelBits; int ptLevelSize = 1 << ptLevelBits;
int ptLevelMask = ptLevelSize - 1; int ptLevelMask = ptLevelSize - 1;
Operand addrRotated = size != 0 ? context.RotateRight(address, Const(size)) : address;
Operand addrShifted = context.ShiftRightUI(addrRotated, Const(PageBits - size));
Operand pte = Ptc.State == PtcState.Disabled Operand pte = Ptc.State == PtcState.Disabled
? Const(context.Memory.PageTablePointer.ToInt64()) ? Const(context.Memory.PageTablePointer.ToInt64())
: Const(context.Memory.PageTablePointer.ToInt64(), true, Ptc.PageTablePointerIndex); : Const(context.Memory.PageTablePointer.ToInt64(), true, Ptc.PageTablePointerIndex);
int bit = PageBits; Operand pteOffset = context.BitwiseAnd(addrShifted, Const(addrShifted.Type, ptLevelMask));
// Load page table entry from the page table.
// This was designed to support multi-level page tables of any size, however right
// now we only use flat page tables (so there's only one level).
// The page table entry contains the host address where the page is located.
// Additionally, the higher 16-bits of the host address may contain extra information
// used for write tracking, so this must be handled here aswell.
do
{
Operand addrPart = context.ShiftRightUI(address, Const(bit));
bit += ptLevelBits;
if (bit < context.Memory.AddressSpaceBits)
{
addrPart = context.BitwiseAnd(addrPart, Const(addrPart.Type, ptLevelMask));
}
Operand pteOffset = context.ShiftLeft(addrPart, Const(3));
if (pteOffset.Type == OperandType.I32) if (pteOffset.Type == OperandType.I32)
{ {
pteOffset = context.ZeroExtend32(OperandType.I64, pteOffset); pteOffset = context.ZeroExtend32(OperandType.I64, pteOffset);
} }
Operand pteAddress = context.Add(pte, pteOffset); pte = context.Load(OperandType.I64, context.Add(pte, context.ShiftLeft(pteOffset, Const(3))));
pte = context.Load(OperandType.I64, pteAddress); if (addrShifted.Type == OperandType.I32)
{
addrShifted = context.ZeroExtend32(OperandType.I64, addrShifted);
} }
while (bit < context.Memory.AddressSpaceBits);
// If the VA is out of range, or not aligned to the access size, force PTE to 0 by masking it.
pte = context.BitwiseAnd(pte, context.ShiftRightSI(context.Add(addrShifted, Const(-(long)ptLevelSize)), Const(63)));
if (lblSlowPath != null) if (lblSlowPath != null)
{ {
ulong protection = (write ? 3UL : 1UL) << 48; if (write)
context.BranchIfTrue(lblSlowPath, context.BitwiseAnd(pte, Const(protection))); {
pte = context.ShiftLeft(pte, Const(1));
context.BranchIf(lblSlowPath, pte, Const(0L), Comparison.LessOrEqual);
pte = context.ShiftRightUI(pte, Const(1));
}
else
{
context.BranchIf(lblSlowPath, pte, Const(0L), Comparison.LessOrEqual);
pte = context.BitwiseAnd(pte, Const(0xffffffffffffUL)); // Ignore any software protection bits. (they are still used by C# memory access)
}
} }
else else
{ {
@ -401,13 +351,15 @@ namespace ARMeilleure.Instructions
Operand lblNotWatched = Label(); Operand lblNotWatched = Label();
// Is the page currently being tracked for read/write? If so we need to call MarkRegionAsModified. // Is the page currently being tracked for read/write? If so we need to call SignalMemoryTracking.
context.BranchIf(lblNotWatched, pte, Const(0L), Comparison.GreaterOrEqual, BasicBlockFrequency.Cold); context.BranchIf(lblNotWatched, pte, Const(0L), Comparison.GreaterOrEqual, BasicBlockFrequency.Cold);
// Mark the region as modified. Size here doesn't matter as address is assumed to be size aligned here. // Signal memory tracking. Size here doesn't matter as address is assumed to be size aligned here.
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.SignalMemoryTracking)), address, Const(1UL), Const(write ? 1 : 0)); context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.SignalMemoryTracking)), address, Const(1UL), Const(write ? 1 : 0));
context.MarkLabel(lblNotWatched); context.MarkLabel(lblNotWatched);
pte = context.BitwiseAnd(pte, Const(0xffffffffffffUL)); // Ignore any software protection bits. (they are still used by C# memory access)
Operand lblNonNull = Label(); Operand lblNonNull = Label();
// Skip exception if the PTE address is non-null (not zero). // Skip exception if the PTE address is non-null (not zero).
@ -418,8 +370,6 @@ namespace ARMeilleure.Instructions
context.MarkLabel(lblNonNull); context.MarkLabel(lblNonNull);
} }
pte = context.BitwiseAnd(pte, Const(0xffffffffffffUL)); // Ignore any software protection bits. (they are still used by c# memory access)
Operand pageOffset = context.BitwiseAnd(address, Const(address.Type, PageMask)); Operand pageOffset = context.BitwiseAnd(address, Const(address.Type, PageMask));
if (pageOffset.Type == OperandType.I32) if (pageOffset.Type == OperandType.I32)

2
ARMeilleure/Translation/PTC/Ptc.cs
View File

@ -22,7 +22,7 @@ namespace ARMeilleure.Translation.PTC
{ {
private const string HeaderMagic = "PTChd"; private const string HeaderMagic = "PTChd";
private const int InternalVersion = 1971; //! To be incremented manually for each change to the ARMeilleure project. private const int InternalVersion = 1987; //! To be incremented manually for each change to the ARMeilleure project.
private const string ActualDir = "0"; private const string ActualDir = "0";
private const string BackupDir = "1"; private const string BackupDir = "1";

13
Ryujinx.Cpu/MemoryManager.cs
View File

@ -21,6 +21,8 @@ namespace Ryujinx.Cpu
private const int PteSize = 8; private const int PteSize = 8;
private const int PointerTagBit = 62;
private readonly InvalidAccessHandler _invalidAccessHandler; private readonly InvalidAccessHandler _invalidAccessHandler;
/// <summary> /// <summary>
@ -556,11 +558,12 @@ namespace Ryujinx.Cpu
// Protection is inverted on software pages, since the default value is 0. // Protection is inverted on software pages, since the default value is 0.
protection = (~protection) & MemoryPermission.ReadAndWrite; protection = (~protection) & MemoryPermission.ReadAndWrite;
long tag = (long)protection << 48; long tag = protection switch
if (tag > 0)
{ {
tag |= long.MinValue; // If any protection is present, the whole pte is negative. MemoryPermission.None => 0L,
} MemoryPermission.Read => 2L << PointerTagBit,
_ => 3L << PointerTagBit
};
ulong endVa = (va + size + PageMask) & ~(ulong)PageMask; ulong endVa = (va + size + PageMask) & ~(ulong)PageMask;
long invTagMask = ~(0xffffL << 48); long invTagMask = ~(0xffffL << 48);
@ -628,7 +631,7 @@ namespace Ryujinx.Cpu
// tracking using host guard pages in future, but also supporting platforms where this is not possible. // tracking using host guard pages in future, but also supporting platforms where this is not possible.
// Write tag includes read protection, since we don't have any read actions that aren't performed before write too. // Write tag includes read protection, since we don't have any read actions that aren't performed before write too.
long tag = (write ? 3L : 1L) << 48; long tag = (write ? 3L : 2L) << PointerTagBit;
ulong endVa = (va + size + PageMask) & ~(ulong)PageMask; ulong endVa = (va + size + PageMask) & ~(ulong)PageMask;