diff --git a/src/coreclr/src/jit/hwintrinsic.cpp b/src/coreclr/src/jit/hwintrinsic.cpp index 16332b859894b..ffa9cd5db1259 100644 --- a/src/coreclr/src/jit/hwintrinsic.cpp +++ b/src/coreclr/src/jit/hwintrinsic.cpp @@ -646,24 +646,26 @@ static bool isSupportedBaseType(NamedIntrinsic intrinsic, var_types baseType) (intrinsic == NI_Vector256_get_Zero) || (intrinsic == NI_Vector256_GetElement) || (intrinsic == NI_Vector256_WithElement) || (intrinsic == NI_Vector256_GetLower) || (intrinsic == NI_Vector256_ToScalar)); -#else - assert((intrinsic == NI_Vector64_AsByte) || (intrinsic == NI_Vector64_AsInt16) || - (intrinsic == NI_Vector64_AsInt32) || (intrinsic == NI_Vector64_AsSByte) || +#endif // TARGET_XARCH +#ifdef TARGET_ARM64 + assert((intrinsic == NI_Vector64_AsByte) || (intrinsic == NI_Vector64_AsDouble) || + (intrinsic == NI_Vector64_AsInt16) || (intrinsic == NI_Vector64_AsInt32) || + (intrinsic == NI_Vector64_AsInt64) || (intrinsic == NI_Vector64_AsSByte) || (intrinsic == NI_Vector64_AsSingle) || (intrinsic == NI_Vector64_AsUInt16) || - (intrinsic == NI_Vector64_AsUInt32) || (intrinsic == NI_Vector64_get_AllBitsSet) || - (intrinsic == NI_Vector64_get_Count) || (intrinsic == NI_Vector64_get_Zero) || - (intrinsic == NI_Vector64_GetElement) || (intrinsic == NI_Vector64_ToScalar) || - (intrinsic == NI_Vector64_ToVector128) || (intrinsic == NI_Vector64_ToVector128Unsafe) || - (intrinsic == NI_Vector128_As) || (intrinsic == NI_Vector128_AsByte) || - (intrinsic == NI_Vector128_AsDouble) || (intrinsic == NI_Vector128_AsInt16) || - (intrinsic == NI_Vector128_AsInt32) || (intrinsic == NI_Vector128_AsInt64) || - (intrinsic == NI_Vector128_AsSByte) || (intrinsic == NI_Vector128_AsSingle) || - (intrinsic == NI_Vector128_AsUInt16) || (intrinsic == NI_Vector128_AsUInt32) || - (intrinsic == NI_Vector128_AsUInt64) || (intrinsic == NI_Vector128_get_AllBitsSet) || - (intrinsic == NI_Vector128_get_Count) || (intrinsic == NI_Vector128_get_Zero) || - (intrinsic == NI_Vector128_GetElement) || (intrinsic == NI_Vector128_GetLower) || - (intrinsic == NI_Vector128_ToScalar)); -#endif + (intrinsic == NI_Vector64_AsUInt32) || (intrinsic == NI_Vector64_AsUInt64) || + (intrinsic == NI_Vector64_get_AllBitsSet) || (intrinsic == NI_Vector64_get_Count) || + (intrinsic == NI_Vector64_get_Zero) || (intrinsic == NI_Vector64_GetElement) || + (intrinsic == NI_Vector64_ToScalar) || (intrinsic == NI_Vector64_ToVector128) || + (intrinsic == NI_Vector64_ToVector128Unsafe) || (intrinsic == NI_Vector128_As) || + (intrinsic == NI_Vector128_AsByte) || (intrinsic == NI_Vector128_AsDouble) || + (intrinsic == NI_Vector128_AsInt16) || (intrinsic == NI_Vector128_AsInt32) || + (intrinsic == NI_Vector128_AsInt64) || (intrinsic == NI_Vector128_AsSByte) || + (intrinsic == NI_Vector128_AsSingle) || (intrinsic == NI_Vector128_AsUInt16) || + (intrinsic == NI_Vector128_AsUInt32) || (intrinsic == NI_Vector128_AsUInt64) || + (intrinsic == NI_Vector128_get_AllBitsSet) || (intrinsic == NI_Vector128_get_Count) || + (intrinsic == NI_Vector128_get_Zero) || (intrinsic == NI_Vector128_GetElement) || + (intrinsic == NI_Vector128_GetLower) || (intrinsic == NI_Vector128_ToScalar)); +#endif // TARGET_ARM64 return false; } diff --git a/src/coreclr/src/jit/hwintrinsicarm64.cpp b/src/coreclr/src/jit/hwintrinsicarm64.cpp index dc7bbcab49f04..f777076cb3313 100644 --- a/src/coreclr/src/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/src/jit/hwintrinsicarm64.cpp @@ -359,12 +359,15 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, switch (intrinsic) { case NI_Vector64_AsByte: + case NI_Vector64_AsDouble: case NI_Vector64_AsInt16: case NI_Vector64_AsInt32: + case NI_Vector64_AsInt64: case NI_Vector64_AsSByte: case NI_Vector64_AsSingle: case NI_Vector64_AsUInt16: case NI_Vector64_AsUInt32: + case NI_Vector64_AsUInt64: case NI_Vector128_As: case NI_Vector128_AsByte: case NI_Vector128_AsDouble: diff --git a/src/coreclr/src/jit/hwintrinsiclistarm64.h b/src/coreclr/src/jit/hwintrinsiclistarm64.h index be8a0d552da75..bee94748b4a8c 100644 --- a/src/coreclr/src/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/src/jit/hwintrinsiclistarm64.h @@ -17,12 +17,15 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Vector64 Intrinsics HARDWARE_INTRINSIC(Vector64, AsByte, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector64, AsDouble, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, AsInt16, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, AsInt32, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector64, AsInt64, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, AsSByte, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, AsSingle, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, AsUInt16, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, AsUInt32, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector64, AsUInt64, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, Create, 8, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_mov, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector64, CreateScalarUnsafe, 8, 1, {INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_invalid, INS_invalid, INS_fmov, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_SupportsContainment|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector64, get_AllBitsSet, 8, 0, {INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs index f2b4fe0ae6d9d..bb78662d38f03 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs @@ -935,45 +935,78 @@ static Vector128 SoftwareFallback(ulong e0, ulong e1) /// The value that the lower 64-bits will be initialized to. /// The value that the upper 64-bits will be initialized to. /// A new initialized from and . + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } + + return SoftwareFallback(lower, upper); + + static Vector128 SoftwareFallback(Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; - return result128; + return result128; + } } /// Creates a new instance from two instances. /// The value that the lower 64-bits will be initialized to. /// The value that the upper 64-bits will be initialized to. /// A new initialized from and . + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } + + return SoftwareFallback(lower, upper); - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + static Vector128 SoftwareFallback (Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; + + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; - return result128; + return result128; + } } /// Creates a new instance from two instances. /// The value that the lower 64-bits will be initialized to. /// The value that the upper 64-bits will be initialized to. /// A new initialized from and . + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + return SoftwareFallback(lower, upper); - return result128; + static Vector128 SoftwareFallback(Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; + + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; + + return result128; + } } /// Creates a new instance from two instances. @@ -981,30 +1014,52 @@ public static unsafe Vector128 Create(Vector64 lower, Vector64The value that the upper 64-bits will be initialized to. /// On x86, this method corresponds to __m128i _mm_setr_epi64 /// A new initialized from and . + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + return SoftwareFallback(lower, upper); + + static Vector128 SoftwareFallback(Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; - return result128; + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; + + return result128; + } } /// Creates a new instance from two instances. /// The value that the lower 64-bits will be initialized to. /// The value that the upper 64-bits will be initialized to. /// A new initialized from and . + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } + + return SoftwareFallback(lower, upper); + + static Vector128 SoftwareFallback(Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; - return result128; + return result128; + } } /// Creates a new instance from two instances. @@ -1012,30 +1067,52 @@ public static unsafe Vector128 Create(Vector64 lower, Vector64 /// The value that the upper 64-bits will be initialized to. /// A new initialized from and . [CLSCompliant(false)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } + + return SoftwareFallback(lower, upper); - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + static Vector128 SoftwareFallback(Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; - return result128; + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; + + return result128; + } } /// Creates a new instance from two instances. /// The value that the lower 64-bits will be initialized to. /// The value that the upper 64-bits will be initialized to. /// A new initialized from and . + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + return SoftwareFallback(lower, upper); - return result128; + static Vector128 SoftwareFallback(Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; + + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; + + return result128; + } } /// Creates a new instance from two instances. @@ -1043,15 +1120,26 @@ public static unsafe Vector128 Create(Vector64 lower, Vector64The value that the upper 64-bits will be initialized to. /// A new initialized from and . [CLSCompliant(false)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } + + return SoftwareFallback(lower, upper); + + static Vector128 SoftwareFallback(Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; - return result128; + return result128; + } } /// Creates a new instance from two instances. @@ -1060,15 +1148,26 @@ public static unsafe Vector128 Create(Vector64 lower, Vector64On x86, this method corresponds to __m128i _mm_setr_epi64 /// A new initialized from and . [CLSCompliant(false)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + return SoftwareFallback(lower, upper); - return result128; + static Vector128 SoftwareFallback(Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; + + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; + + return result128; + } } /// Creates a new instance from two instances. @@ -1076,15 +1175,26 @@ public static unsafe Vector128 Create(Vector64 lower, Vector64 /// The value that the upper 64-bits will be initialized to. /// A new initialized from and . [CLSCompliant(false)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe Vector128 Create(Vector64 lower, Vector64 upper) { - Vector128 result128 = Vector128.Zero; + if (AdvSimd.IsSupported) + { + return lower.ToVector128Unsafe().WithUpper(upper); + } + + return SoftwareFallback(lower, upper); + + static Vector128 SoftwareFallback(Vector64 lower, Vector64 upper) + { + Vector128 result128 = Vector128.Zero; - ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); - result64 = lower; - Unsafe.Add(ref result64, 1) = upper; + ref Vector64 result64 = ref Unsafe.As, Vector64>(ref result128); + result64 = lower; + Unsafe.Add(ref result64, 1) = upper; - return result128; + return result128; + } } /// Creates a new instance with the first element initialized to the specified value and the remaining elements initialized to zero. @@ -1568,14 +1678,28 @@ public static Vector64 GetLower(this Vector128 vector) /// The value of the lower 64-bits as a . /// A new with the lower 64-bits set to and the upper 64-bits set to the same value as that in . /// The type of () is not supported. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 WithLower(this Vector128 vector, Vector64 value) where T : struct { - ThrowHelper.ThrowForUnsupportedVectorBaseType(); + if (AdvSimd.IsSupported) + { + // Note: The 3rd operand GetElement() should be the argument to Insert(). Storing the + // result of GetElement() in a local variable and then passing local variable to Insert() + // would not merge insert/getelement in a single instruction. + return AdvSimd.Insert(vector.AsUInt64(), 0, value.AsUInt64().GetElement(0)).As(); + } - Vector128 result = vector; - Unsafe.As, Vector64>(ref result) = value; - return result; + return SoftwareFallback(vector, value); + + static Vector128 SoftwareFallback(Vector128 vector, Vector64 value) + { + ThrowHelper.ThrowForUnsupportedVectorBaseType(); + + Vector128 result = vector; + Unsafe.As, Vector64>(ref result) = value; + return result; + } } /// Gets the value of the upper 64-bits as a new . @@ -1598,15 +1722,29 @@ public static Vector64 GetUpper(this Vector128 vector) /// The value of the upper 64-bits as a . /// A new with the upper 64-bits set to and the lower 64-bits set to the same value as that in . /// The type of () is not supported. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 WithUpper(this Vector128 vector, Vector64 value) where T : struct { - ThrowHelper.ThrowForUnsupportedVectorBaseType(); + if (AdvSimd.IsSupported) + { + // Note: The 3rd operand GetElement() should be the argument to Insert(). Storing the + // result of GetElement() in a local variable and then passing local variable to Insert() + // would not merge insert/getelement in a single instruction. + return AdvSimd.Insert(vector.AsUInt64(), 1, value.AsUInt64().GetElement(0)).As(); + } - Vector128 result = vector; - ref Vector64 lower = ref Unsafe.As, Vector64>(ref result); - Unsafe.Add(ref lower, 1) = value; - return result; + return SoftwareFallback(vector, value); + + static Vector128 SoftwareFallback(Vector128 vector, Vector64 value) + { + ThrowHelper.ThrowForUnsupportedVectorBaseType(); + + Vector128 result = vector; + ref Vector64 lower = ref Unsafe.As, Vector64>(ref result); + Unsafe.Add(ref lower, 1) = value; + return result; + } } /// Converts the given vector to a scalar containing the value of the first element.