using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Text; using UnityEngine; namespace UnityEditor.Build.Pipeline.Utilities { /// /// Stores hash information as an array of bytes. /// [Serializable] public struct RawHash : IEquatable { readonly byte[] m_Hash; internal RawHash(byte[] hash) { m_Hash = hash; } internal static RawHash Zero() { return new RawHash(new byte[16]); } /// /// Converts the hash to bytes. /// /// Returns the converted hash as an array of bytes. public byte[] ToBytes() { return m_Hash; } /// /// Converts the hash to format. /// /// Returns the converted hash. public Hash128 ToHash128() { if (m_Hash == null || m_Hash.Length != 16) return new Hash128(); return new Hash128(BitConverter.ToUInt32(m_Hash, 0), BitConverter.ToUInt32(m_Hash, 4), BitConverter.ToUInt32(m_Hash, 8), BitConverter.ToUInt32(m_Hash, 12)); } /// /// Converts the hash to a guid. /// /// Returns the converted hash as a guid. public GUID ToGUID() { if (m_Hash == null || m_Hash.Length != 16) return new GUID(); return new GUID(ToString()); } /// /// Converts the hash to a formatted string. /// /// Returns the hash as a string. public override string ToString() { if (m_Hash == null || m_Hash.Length != 16) return "00000000000000000000000000000000"; return BitConverter.ToString(m_Hash).Replace("-", "").ToLower(); } /// /// Determines if the current hash instance is equivalent to the specified hash. /// /// The hash to compare to. /// Returns true if the hashes are equivalent. Returns false otherwise. public bool Equals(RawHash other) { return m_Hash.SequenceEqual(other.m_Hash); } /// /// Determines if the current hash instance is equivalent to the specified hash. /// /// The hash to compare to. /// Returns true if the hashes are equivalent. Returns false otherwise. public override bool Equals(object obj) { if (ReferenceEquals(null, obj)) return false; return obj is RawHash && Equals((RawHash)obj); } /// /// Creates the hash code for the cache entry. /// /// Returns the hash code for the cache entry. public override int GetHashCode() { return (m_Hash != null ? m_Hash.GetHashCode() : 0); } /// /// Determines if the left hash instance is equivalent to the right hash. /// /// The hash to compare against. /// The hash to compare to. /// Returns true if the hashes are equivalent. Returns false otherwise. public static bool operator==(RawHash left, RawHash right) { return left.Equals(right); } /// /// Determines if the left hash instance is not equivalent to the right hash. /// /// The hash to compare against. /// The hash to compare to. /// Returns true if the hashes are not equivalent. Returns false otherwise. public static bool operator!=(RawHash left, RawHash right) { return !(left == right); } } /// /// Creates the for an object. /// public static class HashingMethods { // TODO: Make this even faster! // Maybe use unsafe code to access the raw bytes and pass them directly into the stream? // Maybe pass the bytes into the HashAlgorithm directly // TODO: Does this handle arrays? static void GetRawBytes(Stack state, Stream stream) { if (state.Count == 0) return; object currObj = state.Pop(); if (currObj == null) return; // Handle basic types if (currObj is bool) { var bytes = BitConverter.GetBytes((bool)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is char) { var bytes = BitConverter.GetBytes((char)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is double) { var bytes = BitConverter.GetBytes((double)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is short) { var bytes = BitConverter.GetBytes((short)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is int) { var bytes = BitConverter.GetBytes((int)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is long) { var bytes = BitConverter.GetBytes((long)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is float) { var bytes = BitConverter.GetBytes((float)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is ushort) { var bytes = BitConverter.GetBytes((ushort)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is uint) { var bytes = BitConverter.GetBytes((uint)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is ulong) { var bytes = BitConverter.GetBytes((ulong)currObj); stream.Write(bytes, 0, bytes.Length); } else if (currObj is byte[]) { var bytes = (byte[])currObj; stream.Write(bytes, 0, bytes.Length); } else if (currObj is string) { byte[] bytes; var str = (string)currObj; if (str.Any(c => c > 127)) bytes = Encoding.Unicode.GetBytes(str); else bytes = Encoding.ASCII.GetBytes(str); stream.Write(bytes, 0, bytes.Length); } else if (currObj is GUID) { byte[] hashBytes = new byte[16]; IntPtr ptr = Marshal.AllocHGlobal(16); Marshal.StructureToPtr((GUID)currObj, ptr, false); Marshal.Copy(ptr, hashBytes, 0, 16); Marshal.FreeHGlobal(ptr); PassBytesToStreamInBlocks(stream, hashBytes, 4); } else if (currObj is Hash128) { byte[] hashBytes = new byte[16]; IntPtr ptr = Marshal.AllocHGlobal(16); Marshal.StructureToPtr((Hash128)currObj, ptr, false); Marshal.Copy(ptr, hashBytes, 0, 16); Marshal.FreeHGlobal(ptr); PassBytesToStreamInBlocks(stream, hashBytes, 4); } else if (currObj.GetType().IsEnum) { // Handle enums var type = Enum.GetUnderlyingType(currObj.GetType()); var newObj = Convert.ChangeType(currObj, type); state.Push(newObj); } else if (currObj.GetType().IsArray) { // Handle arrays var array = (Array)currObj; for (int i = array.Length - 1; i >= 0; i--) state.Push(array.GetValue(i)); } else if (currObj is System.Collections.IEnumerable) { var iterator = (System.Collections.IEnumerable)currObj; var reverseOrder = new Stack(); foreach (var newObj in iterator) reverseOrder.Push(newObj); foreach (var newObj in reverseOrder) state.Push(newObj); } else { // Use reflection for remainder var fields = currObj.GetType().GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); for (var index = fields.Length - 1; index >= 0; index--) { var field = fields[index]; var newObj = field.GetValue(currObj); state.Push(newObj); } } } static void GetRawBytes(Stream stream, object obj) { var objStack = new Stack(); objStack.Push(obj); while (objStack.Count > 0) GetRawBytes(objStack, stream); } static void GetRawBytes(Stream stream, params object[] objects) { var objStack = new Stack(); for (var index = objects.Length - 1; index >= 0; index--) objStack.Push(objects[index]); while (objStack.Count > 0) GetRawBytes(objStack, stream); } internal static HashAlgorithm GetHashAlgorithm() { #if UNITY_2020_1_OR_NEWER // New projects on 2021.1 will default useSpookyHash to true // Upgraded projects will remain false until they choose to switch return ScriptableBuildPipeline.useV2Hasher ? (HashAlgorithm)SpookyHash.Create() : new MD5CryptoServiceProvider(); #else return new MD5CryptoServiceProvider(); #endif } internal static HashAlgorithm GetHashAlgorithm(Type type) { if (type == null) { #if UNITY_2020_1_OR_NEWER // New projects on 2021.1 will default useSpookyHash to true // Upgraded projects will remain false until they choose to switch type = ScriptableBuildPipeline.useV2Hasher ? typeof(SpookyHash) : typeof(MD5); #else type = typeof(MD5); #endif } if (type == typeof(MD4)) return MD4.Create(); #if UNITY_2019_3_OR_NEWER if (type == typeof(SpookyHash)) return SpookyHash.Create(); #endif // TODO: allow user created HashAlgorithms? var alggorithm = HashAlgorithm.Create(type.FullName); if (alggorithm == null) throw new NotImplementedException("Unable to create hash algorithm: '" + type.FullName + "'."); return alggorithm; } /// /// Creates the hash for a stream of data. /// /// The stream of data. /// Returns the hash of the stream. public static RawHash CalculateStream(Stream stream) { if (stream == null) return RawHash.Zero(); if (stream is HashStream hs) return hs.GetHash(); byte[] hash; using (var hashAlgorithm = GetHashAlgorithm()) hash = hashAlgorithm.ComputeHash(stream); return new RawHash(hash); } /// /// Creates the hash for a stream of data. /// /// The hash algorithm type. /// The stream of data. /// Returns the hash of the stream. public static RawHash CalculateStream(Stream stream) where T : HashAlgorithm { if (stream == null) return RawHash.Zero(); if (stream is HashStream hs) return hs.GetHash(); byte[] hash; using (var hashAlgorithm = GetHashAlgorithm(typeof(T))) hash = hashAlgorithm.ComputeHash(stream); return new RawHash(hash); } /// /// Creates the hash for an object. /// /// The object. /// Returns the hash of the object. public static RawHash Calculate(object obj) { RawHash rawHash; using (var stream = new HashStream(GetHashAlgorithm())) { GetRawBytes(stream, obj); rawHash = stream.GetHash(); } return rawHash; } /// /// Creates the hash for a set of objects. /// /// The objects. /// Returns the hash of the set of objects. public static RawHash Calculate(params object[] objects) { if (objects == null) return RawHash.Zero(); RawHash rawHash; using (var stream = new HashStream(GetHashAlgorithm())) { GetRawBytes(stream, objects); rawHash = stream.GetHash(); } return rawHash; } /// /// Creates the hash for a pair of Hash128 objects. Optimized specialization of the generic Calculate() methods that has been shown to be ~3x faster /// The generic function uses reflection to obtain the four 32bit fields in the Hash128 which is slow, this function uses more direct byte access /// /// The first hash to combine /// The second hash to combine /// Returns the combined hash of the two hashes. public static RawHash Calculate(Hash128 hash1, Hash128 hash2) { byte[] hashBytes = new byte[32]; IntPtr ptr = Marshal.AllocHGlobal(16); Marshal.StructureToPtr(hash1, ptr, false); Marshal.Copy(ptr, hashBytes, 0, 16); Marshal.StructureToPtr(hash2, ptr, true); Marshal.Copy(ptr, hashBytes, 16, 16); Marshal.FreeHGlobal(ptr); HashStream hashStream = new HashStream(GetHashAlgorithm()); PassBytesToStreamInBlocks(hashStream, hashBytes, 4); return hashStream.GetHash(); } /// /// Send bytes from an array to a stream as a sequence of blocks /// Not all hash algorithms behave the same passing data as one large block instead of multiple smaller ones so produce different hashes if /// we pass a GUID or Hash128 as a single 16 byte block for example instead of the 4x4 byte blocks the generic hashing code produces for these types /// This function passes bytes from a buffer in chunks of a specified size to ensure we get the same hash from the same data in such cases /// Our SpookyHash implementation suffers from this for example (see SpookyHash::Short()) while MD5 does not /// /// Stream to write bytes to /// Array of bytes to pass to the stream, must be multiple of blockSizeBytes in size or an InvalidOperationException will be thrown /// Number of bytes to write to the stream each time /// static void PassBytesToStreamInBlocks(Stream stream, byte[] bytesToWrite, int blockSizeBytes) { if ((bytesToWrite.Length % blockSizeBytes) != 0) throw new InvalidOperationException($"PassBytesToStreamInBlocks() byte array size of {bytesToWrite.Length} is required to be a multiple of {blockSizeBytes} which it's not"); for (int offset = 0; offset < bytesToWrite.Length; offset += blockSizeBytes) { stream.Write(bytesToWrite, offset, blockSizeBytes); } } /// /// Creates the hash for an object. /// /// The hash algorithm type. /// The object. /// Returns the hash of the object. public static RawHash Calculate(object obj) where T : HashAlgorithm { RawHash rawHash; using (var stream = new HashStream(GetHashAlgorithm(typeof(T)))) { GetRawBytes(stream, obj); rawHash = stream.GetHash(); } return rawHash; } /// /// Creates the hash for a set of objects. /// /// The hash algorithm type. /// The objects. /// Returns the hash of the set of objects. public static RawHash Calculate(params object[] objects) where T : HashAlgorithm { if (objects == null) return RawHash.Zero(); RawHash rawHash; using (var stream = new HashStream(GetHashAlgorithm(typeof(T)))) { GetRawBytes(stream, objects); rawHash = stream.GetHash(); } return rawHash; } /// /// Creates the hash for a file. /// /// The file path. /// Returns the hash of the file. public static RawHash CalculateFile(string filePath) { RawHash rawHash; using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read)) rawHash = CalculateStream(stream); return rawHash; } /// /// Creates the hash for a file. /// /// The hash algorithm type. /// The file path. /// Returns the hash of the file. public static RawHash CalculateFile(string filePath) where T : HashAlgorithm { RawHash rawHash; using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read)) rawHash = CalculateStream(stream); return rawHash; } } }