Just-DataInspector/Utilities/Src/MatH5.cpp

// HDF5 只读子集：MATLAB 7.3 `.mat` 解析用（见 MatH5.hpp 的说明）。
#include "MatH5.hpp"

#include "Deflate.hpp"

#include <windows.h>

#include <algorithm>
#include <cstdio>
#include <cstring>

namespace jdi
{

	namespace
	{
		constexpr std::uint16_t kMessageDataspace = 0x0001;
		constexpr std::uint16_t kMessageLink = 0x0006;
		constexpr std::uint16_t kMessageDatatype = 0x0003;
		constexpr std::uint16_t kMessageLayout = 0x0008;
		constexpr std::uint16_t kMessageFilter = 0x000B;
		constexpr std::uint16_t kMessageAttribute = 0x000C;
		constexpr std::uint16_t kMessageContinuation = 0x0010;
		constexpr std::uint16_t kMessageSymbolTable = 0x0011;

		std::uint64_t ReadLE(const std::uint8_t* data, std::size_t size)
		{
			std::uint64_t value = 0;
			for (std::size_t index = 0; index < size; ++index)
			{
				value |= static_cast<std::uint64_t>(data[index]) << (8 * index);
			}
			return value;
		}

		/** @brief 连续布局的读取切片大小（映射视图下只是 memcpy，不会产生额外峰值）。 */
		constexpr std::size_t kStreamSliceBytes = 4u << 20;

		/** @brief 把一段原始字节按元素类型转换成 double（供流式解码使用）。 */
		void ConvertNumbers(const std::uint8_t* source, std::size_t count, std::size_t elementSize,
		                    bool integer, bool floating, bool isSigned, double* out)
		{
			for (std::size_t index = 0; index < count; ++index)
			{
				const std::uint8_t* bytes = source + index * elementSize;
				if (floating && elementSize == 8)
				{
					double value = 0.0;
					std::memcpy(&value, bytes, sizeof(value));
					out[index] = value;
				}
				else if (floating && elementSize == 4)
				{
					float value = 0.0f;
					std::memcpy(&value, bytes, sizeof(value));
					out[index] = static_cast<double>(value);
				}
				else if (integer && isSigned)
				{
					std::int64_t value = 0;
					for (std::size_t byte = 0; byte < elementSize; ++byte)
					{
						value |= static_cast<std::int64_t>(bytes[byte]) << (8 * byte);
					}
					const int bits = static_cast<int>(elementSize) * 8;
					if (bits < 64 && (value & (1ll << (bits - 1))) != 0)
					{
						value |= ~((1ll << bits) - 1);
					}
					out[index] = static_cast<double>(value);
				}
				else
				{
					std::uint64_t value = 0;
					for (std::size_t byte = 0; byte < elementSize; ++byte)
					{
						value |= static_cast<std::uint64_t>(bytes[byte]) << (8 * byte);
					}
					out[index] = static_cast<double>(value);
				}
			}
		}
	} // namespace

	namespace
	{
		/** @brief 元数据缓存的块大小与槽数（16 × 64KB = 1MB，足够覆盖对象头/B-tree 热点）。 */
		constexpr std::size_t kMetaBlockBytes = 64u * 1024u;
		constexpr std::size_t kMetaBlockSlots = 16;

		/** @brief 顺序预读窗口上限。 */
		constexpr std::size_t kReadAheadBytes = 4u * 1024u * 1024u;

		/**
		 * @brief 数据读取的最小粒度：按请求大小向上对齐读取，命中窗口时直接复用。
		 *
		 * ⚠️ 分块数据集的 chunk 在文件里**并非完全顺序**（多个样本流交错写入，
		 * 每段连续区域约 1.8MB）。实测 `BIG_MAT.mat`：固定预读 4MB 会让每个连续段
		 * 末尾浪费 2.2MB，总读盘量达到文件大小的 **2.2 倍**（2.87GB / 1.29GB）。
		 * 改为「按需 + 64KB 对齐」后，读盘量回到文件大小附近。
		 */
		constexpr std::size_t kReadGranularity = 64u * 1024u;
	}

	bool MatHdf5::ReadAt(std::uint64_t offset, void* dst, std::size_t bytes) const
	{
		if (m_fileHandle == nullptr || bytes == 0)
		{
			return bytes == 0;
		}
		if (offset > m_size || bytes > m_size - offset)
		{
			return false;
		}
		const HANDLE file = static_cast<HANDLE>(m_fileHandle);
		LARGE_INTEGER position = {};
		position.QuadPart = static_cast<LONGLONG>(offset);
		if (!SetFilePointerEx(file, position, nullptr, FILE_BEGIN))
		{
			return false;
		}
		std::size_t done = 0;
		while (done < bytes)
		{
			DWORD read = 0;
			const DWORD want = static_cast<DWORD>(std::min<std::size_t>(bytes - done, 1u << 30));
			if (!ReadFile(file, static_cast<std::uint8_t*>(dst) + done, want, &read, nullptr))
			{
				return false;
			}
			if (read == 0)
			{
				return false;	// 文件被截断
			}
			done += read;
		}
		m_bytesRead += bytes;
		return true;
	}

	const std::uint8_t* MatHdf5::Cached(std::uint64_t offset, std::size_t bytes) const
	{
		if (offset >= m_size || bytes > kMetaBlockBytes)
		{
			return nullptr;
		}
		if (m_cache.empty())
		{
			m_cache.resize(kMetaBlockSlots);
		}
		const std::uint64_t base = offset - (offset % kMetaBlockBytes);
		// 快路径：连续读同一块（U16/U32/U64 与 B-tree 遍历几乎都是这种模式）。
		{
			CacheSlot& slot = m_cache[m_lastSlot];
			if (slot.length > 0 && slot.base == base)
			{
				slot.stamp = ++m_stamp;
				const std::uint64_t delta = offset - slot.base;
				return (delta + bytes <= slot.length) ? (slot.bytes.data() + delta) : nullptr;
			}
		}
		CacheSlot* victim = nullptr;
		for (std::size_t index = 0; index < m_cache.size(); ++index)
		{
			CacheSlot& slot = m_cache[index];
			if (slot.length > 0 && slot.base == base)
			{
				slot.stamp = ++m_stamp;
				m_lastSlot = index;
				const std::uint64_t delta = offset - slot.base;
				return (delta + bytes <= slot.length) ? (slot.bytes.data() + delta) : nullptr;
			}
			if (slot.length == 0 || victim == nullptr || slot.stamp < victim->stamp)
			{
				victim = &slot;
			}
		}
		if (victim == nullptr)
		{
			return nullptr;
		}
		// 未命中：整块读入（块可能超出文件末尾，按可用长度裁剪）。
		const std::size_t wanted =
		    static_cast<std::size_t>(std::min<std::uint64_t>(kMetaBlockBytes, m_size - base));
		if (bytes > wanted)
		{
			return nullptr;
		}
		victim->bytes.resize(wanted);
		victim->base = base;
		victim->length = static_cast<std::uint32_t>(wanted);
		victim->stamp = ++m_stamp;
		m_lastSlot = static_cast<std::size_t>(victim - m_cache.data());
		if (!ReadAt(base, victim->bytes.data(), wanted))
		{
			victim->length = 0;
			return nullptr;
		}
		return victim->bytes.data() + (offset - base);
	}

	bool MatHdf5::Tag(std::uint64_t offset, const char* id) const
	{
		const std::uint8_t* bytes = Cached(offset, 4);
		if (bytes == nullptr)
		{
			return false;
		}
		return std::memcmp(bytes, id, 4) == 0;
	}

	void MatHdf5::ReadCString(std::uint64_t offset, std::string& out) const
	{
		out.clear();
		std::uint64_t position = offset;
		while (position < m_size && out.size() < 4096)
		{
			const std::uint8_t* bytes = Cached(position, 1);
			if (bytes == nullptr)
			{
				return;
			}
			// 只消费本缓存块内剩余的字节，保证下一次循环拿到的新块仍然有效。
			const std::uint64_t blockBase = position - (position % kMetaBlockBytes);
			const std::size_t limit =
			    static_cast<std::size_t>(std::min<std::uint64_t>(kMetaBlockBytes - (position - blockBase),
			                                                     m_size - position));
			std::size_t used = 0;
			while (used < limit)
			{
				if (bytes[used] == '\0')
				{
					return;
				}
				out.push_back(static_cast<char>(bytes[used]));
				++used;
			}
			position += used;
		}
	}

	void MatHdf5::DropReadAhead() const
	{
		m_readAheadValid = false;
		m_readAheadLength = 0;
	}

	const std::uint8_t* MatHdf5::ReadData(std::uint64_t offset, std::size_t bytes) const
	{
		if (bytes > kReadAheadBytes || offset > m_size || bytes > m_size - offset)
		{
			return nullptr;
		}
		if (m_readAheadValid && offset >= m_readAheadBase &&
		    (offset - m_readAheadBase) + bytes <= m_readAheadLength)
		{
			return m_readAhead.data() + (offset - m_readAheadBase);
		}
		// 按需读取 + 64KB 对齐：命中当前窗口就直接复用，否则只读「这次需要的一段」。
		// （固定 4MB 预读在交错存放的分块数据集上会浪费 2 倍以上的 I/O。）
		const std::size_t requested = std::max<std::size_t>(bytes, kReadGranularity);
		std::size_t wanted = static_cast<std::size_t>(std::min<std::uint64_t>(
		    m_size - offset, std::min<std::size_t>(kReadAheadBytes,
		                                           ((requested + kReadGranularity - 1) / kReadGranularity) *
		                                               kReadGranularity)));
		if (m_readAhead.size() < wanted)
		{
			m_readAhead.resize(wanted);
		}
		if (!ReadAt(offset, m_readAhead.data(), wanted))
		{
			DropReadAhead();
			return nullptr;
		}
		m_readAheadBase = offset;
		m_readAheadLength = static_cast<std::uint32_t>(wanted);
		m_readAheadValid = true;
		if (m_readProgress && !m_readProgress(m_bytesRead, m_size))
		{
			m_readCancelled = true;
			DropReadAhead();
			return nullptr;
		}
		return m_readAhead.data();
	}

	std::uint8_t MatHdf5::U8(std::uint64_t offset) const
	{
		const std::uint8_t* bytes = Cached(offset, 1);
		return (bytes != nullptr) ? *bytes : 0;
	}

	std::uint16_t MatHdf5::U16(std::uint64_t offset) const
	{
		return static_cast<std::uint16_t>(U8(offset)) | (static_cast<std::uint16_t>(U8(offset + 1)) << 8);
	}

	std::uint32_t MatHdf5::U32(std::uint64_t offset) const
	{
		return static_cast<std::uint32_t>(U16(offset)) | (static_cast<std::uint32_t>(U16(offset + 2)) << 16);
	}

	std::uint64_t MatHdf5::U64(std::uint64_t offset) const
	{
		return static_cast<std::uint64_t>(U32(offset)) | (static_cast<std::uint64_t>(U32(offset + 4)) << 32);
	}

	std::uint64_t MatHdf5::Addr(std::uint64_t offset) const
	{
		return (m_offsetSize == 8) ? U64(offset) : U32(offset);
	}

	std::uint64_t MatHdf5::Resolve(std::uint64_t address) const
	{
		if (address == 0 || address == 0xFFFFFFFFFFFFFFFFull)
		{
			return 0;
		}
		return address + m_base;
	}

	bool MatHdf5::Open(const std::wstring& path, std::wstring& error, const OpenProgress& progress)
	{
		error.clear();
		Close();

		// ⚠️ **随机读，不映射**：SDI 导出的 `.mat` 可能是 GB 级，整文件读入会同时驻留
		//「文件 + 未压缩字节 + double」三份内存；而内存映射在文件被截断 / 网络共享掉线 /
		// 移动介质拔出时会触发 SEH（页访问异常），GCC 下的 `try/catch` 捕获不到 ⇒ 进程直接崩溃。
		// 这里改用 `ReadFile` + 元数据块缓存：任何 I/O 失败都只会变成 `false`。
		const HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr,
		                                OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,
		                                nullptr);
		if (file == INVALID_HANDLE_VALUE)
		{
			error = L"无法打开文件。";
			return false;
		}
		LARGE_INTEGER fileSize = {};
		if (!GetFileSizeEx(file, &fileSize) || fileSize.QuadPart <= 0)
		{
			CloseHandle(file);
			error = L"无法读取文件大小。";
			return false;
		}
		m_fileHandle = file;
		m_size = static_cast<std::uint64_t>(fileSize.QuadPart);
		m_bytesRead = 0;
		m_readCancelled = false;

		// 打开阶段已知道总大小：立即上报一次，遮罩层可以显示「0 / 总大小」，
		// 之后的真实进度由 `ReadAt` 累计的已读字节数驱动。
		if (progress && !progress(0, m_size))
		{
			Close();
			error.clear();	// 用户取消：不显示错误
			return false;
		}

		// 超级块：签名可能出现在 0、512、1024…（MATLAB 用 512 字节 user block）。
		std::size_t signature = static_cast<std::size_t>(m_size);
		for (std::size_t index = 0; index + 8 <= m_size && index <= 4096; index += 512)
		{
			if (Tag(index, "\x89HDF"))
			{
				signature = index;
				break;
			}
		}
		if (signature == m_size)
		{
			error = L"不是 HDF5（MATLAB 7.3）格式。";
			return false;
		}
		const std::uint8_t version = U8(signature + 8);
		if (version == 0 || version == 1)
		{
			m_offsetSize = U8(signature + 13);
			m_lengthSize = U8(signature + 14);
			m_base = U64(signature + 24);
			const std::uint64_t rootEntry = signature + 24 + 32;
			m_root = Resolve(Addr(rootEntry + 8));
		}
		else
		{
			m_offsetSize = U8(signature + 9);
			m_lengthSize = U8(signature + 10);
			m_base = U64(signature + 12);
			m_root = Resolve(U64(signature + 36));
		}
		if (m_root == 0 || m_root >= m_size)
		{
			error = L"HDF5 超级块损坏。";
			return false;
		}
		return true;
	}

	void MatHdf5::Close()
	{
		DropReadAhead();
		m_cache.clear();
		m_lastSlot = 0;
		if (m_fileHandle != nullptr)
		{
			CloseHandle(static_cast<HANDLE>(m_fileHandle));
			m_fileHandle = nullptr;
		}
		m_size = 0;
		m_bytesRead = 0;
		m_readCancelled = false;
		m_base = 0;
		m_root = 0;
		m_heaps.clear();
		m_readProgress = ReadProgress();
	}

	void MatHdf5::ReadMessages(std::uint64_t address, std::vector<Message>& messages) const
	{
		messages.clear();
		if (address + 8 > m_size)
		{
			return;
		}
		if (U8(address) == 1)
		{
			const std::uint16_t count = U16(address + 2);
			const std::uint32_t headerSize = U32(address + 8);
			// 先收集消息块：主块（长度 = 对象头里的 headerSize）+ 若干续块（续块里没有对象头前缀）。
			// ⚠️ **主块长度必须用 headerSize 限定**：消息数超过主块容量时，其余消息在续块里，
			// 若在主块里继续按"消息数"往下读，就会读到块外的其它数据（字段全部解析失败）。
			std::vector<std::pair<std::uint64_t, std::uint64_t>> blocks;
			blocks.emplace_back(address + 16, headerSize);
			std::size_t read = 0;
			for (std::size_t blockIndex = 0; blockIndex < blocks.size(); ++blockIndex)
			{
				std::uint64_t offset = blocks[blockIndex].first;
				const std::uint64_t end =
				    (blocks[blockIndex].second > 0) ? (offset + blocks[blockIndex].second) : 0;
				while (read < count && offset + 8 <= m_size && (end == 0 || offset + 8 <= end))
				{
					Message message;
					message.type = U16(offset);
					const std::uint16_t messageSize = U16(offset + 2);
					offset += 8;
					if (offset + messageSize > m_size)
					{
						break;
					}
					message.data.resize(messageSize);
					if (messageSize > 0 && !ReadAt(offset, message.data.data(), messageSize))
					{
						break;
					}
					offset += (messageSize + 7u) & ~static_cast<std::uint16_t>(7u);
					++read;
					if (message.type == kMessageContinuation && message.data.size() >= 16)
					{
						const std::uint64_t blockAddress = Resolve(ReadLE(message.data.data(), 8));
						const std::uint64_t blockSize = ReadLE(message.data.data() + 8, 8);
						if (blockAddress != 0)
						{
							blocks.emplace_back(blockAddress, blockSize);
						}
					}
					messages.push_back(std::move(message));
				}
			}
		}
	}

	void MatHdf5::WalkBtree(std::uint64_t node, std::uint64_t heapData,
	                        std::vector<std::pair<std::string, std::uint64_t>>& out) const
	{
		int guard = 0;
		while (node != 0 && node + 24 < m_size && guard++ < 100000)
		{
			if (!Tag(node, "TREE"))
			{
				return;
			}
			const std::uint8_t level = U8(node + 5);
			const std::uint16_t entries = U16(node + 6);
			std::uint64_t offset = node + 8 + 8 + 8;
			std::vector<std::uint64_t> children;
			for (std::uint16_t index = 0; index < static_cast<std::uint16_t>(entries + 1u); ++index)
			{
				offset += 8;	// key（分组 B-tree 的 key 是堆偏移）
				if (index < entries)
				{
					children.push_back(Resolve(U64(offset)));
					offset += 8;
				}
			}
			for (const std::uint64_t child : children)
			{
				if (level > 0)
				{
					WalkBtree(child, heapData, out);
				}
				else
				{
					WalkSymbolNode(child, heapData, out);
				}
			}
			node = Resolve(U64(node + 8));	// 右兄弟
		}
	}

	void MatHdf5::WalkSymbolNode(std::uint64_t node, std::uint64_t heapData,
	                             std::vector<std::pair<std::string, std::uint64_t>>& out) const
	{
		if (node == 0 || node + 8 > m_size || !Tag(node, "SNOD") || heapData == 0)
		{
			return;
		}
		const std::uint16_t count = U16(node + 6);
		std::uint64_t offset = node + 8;
		for (std::uint16_t index = 0; index < count; ++index)
		{
			if (offset + 40 > m_size)
			{
				return;
			}
			const std::uint64_t nameOffset = U64(offset);
			const std::uint64_t objectAddress = Resolve(U64(offset + 8));
			offset += 40;
			if (objectAddress == 0 || heapData + nameOffset >= m_size)
			{
				continue;
			}
			std::string name;
			ReadCString(heapData + nameOffset, name);
			out.emplace_back(std::move(name), objectAddress);
		}
	}

	std::vector<std::pair<std::string, std::uint64_t>> MatHdf5::Members(std::uint64_t address) const
	{
		std::vector<std::pair<std::string, std::uint64_t>> out;
		std::vector<Message> messages;
		ReadMessages(address, messages);
		for (const Message& message : messages)
		{
			if (message.type == kMessageSymbolTable && message.data.size() >= 16)
			{
				const std::uint64_t btree = Resolve(ReadLE(message.data.data(), 8));
				const std::uint64_t heap = Resolve(ReadLE(message.data.data() + 8, 8));
				std::uint64_t heapData = 0;
				if (heap + 32 <= m_size && Tag(heap, "HEAP"))
				{
					heapData = Resolve(U64(heap + 24));
					bool known = false;
					for (const auto& entry : m_heaps)
					{
						if (entry.first == heap)
						{
							known = true;
							break;
						}
					}
					if (!known)
					{
						m_heaps.emplace_back(heap, heapData);
					}
				}
				WalkBtree(btree, heapData, out);
			}
			else if (message.type == kMessageLink && message.data.size() >= 8)
			{
				std::size_t offset = 2;
				const std::uint8_t flags = message.data[1];
				if (flags & 0x08)
				{
					offset += 8;
				}
				if (flags & 0x04)
				{
					offset += 8;
				}
				if (flags & 0x10)
				{
					++offset;
				}
				if (offset >= message.data.size())
				{
					continue;
				}
				const std::uint8_t lengthSize = message.data[offset++];
				std::uint64_t nameLength = 0;
				for (std::uint8_t byte = 0; byte < lengthSize && offset < message.data.size(); ++byte)
				{
					nameLength |= static_cast<std::uint64_t>(message.data[offset + byte]) << (8 * byte);
				}
				offset += lengthSize;
				if (offset + nameLength + 8 > message.data.size())
				{
					continue;
				}
				std::string name(reinterpret_cast<const char*>(message.data.data() + offset),
				                 static_cast<std::size_t>(nameLength));
				offset += static_cast<std::size_t>(nameLength);
				const std::uint64_t target = Resolve(ReadLE(message.data.data() + offset, 8));
				out.emplace_back(std::move(name), target);
			}
		}
		return out;
	}

	std::uint64_t MatHdf5::Member(std::uint64_t address, const std::string& name) const
	{
		for (const auto& entry : Members(address))
		{
			if (entry.first == name)
			{
				return entry.second;
			}
		}
		return 0;
	}

	bool MatHdf5::IsDataset(std::uint64_t address) const
	{
		std::vector<Message> messages;
		ReadMessages(address, messages);
		for (const Message& message : messages)
		{
			if (message.type == kMessageDataspace)
			{
				return true;
			}
		}
		return false;
	}

	bool MatHdf5::ParseType(const std::uint8_t* data, std::size_t size, std::size_t offset, Type& out,
	                        std::size_t& consumed) const
	{
		if (offset + 8 > size)
		{
			return false;
		}
		out.version = data[offset] >> 4;
		out.cls = static_cast<TypeClass>(data[offset] & 0x0F);
		const std::uint8_t bits0 = data[offset + 1];
		out.size = ReadLE(data + offset + 4, 4);
		std::size_t cursor = offset + 8;
		if (out.cls == TypeClass::Integer)
		{
			out.bigEndian = (bits0 & 0x01) != 0;
			out.isSigned = (bits0 & 0x08) != 0;
			if (out.version >= 3)
			{
				cursor += 2;	// bit offset
				cursor += 2;	// precision
			}
			else
			{
				cursor += 4;
			}
		}
		else if (out.cls == TypeClass::Float)
		{
			out.bigEndian = (bits0 & 0x01) != 0;
			cursor += 12;
		}
		else if (out.cls == TypeClass::String)
		{
			// 无需额外属性
		}
		else if (out.cls == TypeClass::Reference)
		{
			// 无需额外属性
		}
		else if (out.cls == TypeClass::VariableLength)
		{
			out.isVariableString = (bits0 & 0x0F) == 1;
			std::size_t childConsumed = 0;
			Type child;
			if (ParseType(data, size, cursor, child, childConsumed))
			{
				cursor += childConsumed;
			}
		}
		else if (out.cls == TypeClass::Compound)
		{
			const unsigned members = static_cast<unsigned>(bits0) | (static_cast<unsigned>(data[offset + 2]) << 8);
			for (unsigned index = 0; index < members; ++index)
			{
				if (cursor >= size)
				{
					break;
				}
				const std::size_t nameStart = cursor;
				while (cursor < size && data[cursor] != 0)
				{
					++cursor;
				}
				std::string name(reinterpret_cast<const char*>(data + nameStart), cursor - nameStart);
				++cursor;	// 结束符
				cursor = (cursor + 7) & ~static_cast<std::size_t>(7);
				if (out.version == 3)
				{
					// v3：先维度（1 字节 rank + 4 字节保留），再字节偏移（可变）
				}
				cursor += 4;	// 字节偏移（v1/v2）
				std::size_t childConsumed = 0;
				Type child;
				if (ParseType(data, size, cursor, child, childConsumed))
				{
					cursor += childConsumed;
					out.members.emplace_back(std::move(name), std::move(child));
				}
				else
				{
					break;
				}
			}
		}
		else if (out.cls == TypeClass::Array)
		{
			if (cursor + 4 <= size)
			{
				const std::uint8_t rank = data[cursor];
				cursor += 4;
				cursor += static_cast<std::size_t>(rank) * 4;	// 维度
				cursor += static_cast<std::size_t>(rank) * 4;	// 排列
				std::size_t childConsumed = 0;
				Type child;
				if (ParseType(data, size, cursor, child, childConsumed))
				{
					cursor += childConsumed;
				}
			}
		}
		consumed = cursor - offset;
		return true;
	}

		bool MatHdf5::ReadDatasetInfo(std::uint64_t address, Dataset& out) const
	{
		out = Dataset();
		std::vector<Message> messages;
		ReadMessages(address, messages);
		bool haveSpace = false;
		bool haveType = false;
		std::size_t compactSize = 0;
		std::uint64_t dataAddress = 0;
		std::uint64_t dataSize = 0;
		for (const Message& message : messages)
		{
			if (message.type == kMessageDataspace && message.data.size() >= 8)
			{
				const std::uint8_t version = message.data[0];
				const std::uint8_t rank = message.data[1];
				std::size_t offset = (version == 1) ? 8 : 4;
				out.dims.assign(rank, 0);
				for (std::uint8_t axis = 0; axis < rank && offset + 8 <= message.data.size(); ++axis)
				{
					out.dims[axis] = static_cast<std::size_t>(ReadLE(&message.data[offset], 8));
					offset += 8;
				}
				haveSpace = true;
			}
			else if (message.type == kMessageDatatype)
			{
				std::size_t consumed = 0;
				haveType = ParseType(message.data.data(), message.data.size(), 0, out.type, consumed);
			}
			else if (message.type == kMessageLayout && message.data.size() >= 8)
			{
				const std::uint8_t version = message.data[0];
				const std::uint8_t klass = message.data[1];
				out.layout = static_cast<LayoutClass>(klass);
				if (klass == 0)
				{
					// 紧凑：数据就放在消息里
					if (version == 3)
					{
						compactSize = static_cast<std::size_t>(message.data[2]) |
						              (static_cast<std::size_t>(message.data[3]) << 8);
					}
					else
					{
						compactSize = static_cast<std::size_t>(ReadLE(&message.data[2], 4));
					}
				}
				else if (klass == 1)
				{
					dataAddress = Resolve(ReadLE(&message.data[2], 8));
					dataSize = ReadLE(&message.data[10], 8);
				}
				else if (klass == 2)
				{
					if (version == 3)
					{
						const std::uint8_t rank = message.data[2];
						out.chunkAddress = Resolve(ReadLE(&message.data[3], 8));
						out.chunkDims.assign(rank, 0);
						for (std::uint8_t axis = 0; axis < rank; ++axis)
						{
							out.chunkDims[axis] = ReadLE(&message.data[11 + axis * 4], 4);
						}
					}
					else
					{
						const std::uint8_t rank = message.data[2];
						out.chunkAddress = Resolve(ReadLE(&message.data[3], 8));
						out.chunkDims.assign(rank, 0);
						for (std::uint8_t axis = 0; axis < rank; ++axis)
						{
							out.chunkDims[axis] = ReadLE(&message.data[11 + axis * 4], 4);
						}
					}
				}
			}
			else if (message.type == kMessageFilter && message.data.size() >= 2)
			{
				const int count = message.data[1];
				std::size_t offset = 8;
				for (int index = 0; index < count && offset + 8 <= message.data.size(); ++index)
				{
					const std::uint16_t id = static_cast<std::uint16_t>(message.data[offset]) |
					                         (static_cast<std::uint16_t>(message.data[offset + 1]) << 8);
					const std::uint16_t nameLength = static_cast<std::uint16_t>(message.data[offset + 2]) |
					                                 (static_cast<std::uint16_t>(message.data[offset + 3]) << 8);
					const std::uint16_t values =
					    static_cast<std::uint16_t>(message.data[offset + 6]) |
					    (static_cast<std::uint16_t>(message.data[offset + 7]) << 8);
					offset += 8;
					offset += (nameLength + 7) & ~static_cast<std::size_t>(7);
					offset += static_cast<std::size_t>(values) * 4;
					if (id == 1)
					{
						out.deflate = true;
					}
				}
			}
		}
		if (!haveSpace || !haveType || out.type.size == 0)
		{
			return false;
		}
		out.dataAddress = dataAddress;
		out.dataBytes = dataSize;
		if (out.layout == LayoutClass::Compact)
		{
			// 紧凑数据就在布局消息里（极小），保留一份以便流式遍历时吐出。
			for (const Message& message : messages)
			{
				if (message.type == kMessageLayout)
				{
					const std::size_t offset = (message.data[0] == 3) ? 4 : 8;
					if (offset + compactSize <= message.data.size())
					{
						out.compact.assign(message.data.begin() + static_cast<std::ptrdiff_t>(offset),
						                   message.data.begin() +
						                       static_cast<std::ptrdiff_t>(offset + compactSize));
					}
				}
			}
		}
		out.ok = true;
		return true;
	}

	bool MatHdf5::ReadDataset(std::uint64_t address, Dataset& out) const
	{
		if (!ReadDatasetInfo(address, out))
		{
			return false;
		}
		const std::size_t elementSize = out.type.size;
		const std::uint64_t total = out.ElementCount();
		if (total == 0 || total > static_cast<std::uint64_t>(SIZE_MAX) / elementSize)
		{
			return false;
		}
		out.data.assign(static_cast<std::size_t>(total) * elementSize, 0);
		const bool ok = StreamDataset(
		    out, [&out, elementSize](const std::uint8_t* bytes, std::size_t byteCount,
		                             std::uint64_t elementOffset) -> bool
		    {
			    const std::size_t target = static_cast<std::size_t>(elementOffset) * elementSize;
			    if (target + byteCount > out.data.size())
			    {
				    return false;
			    }
			    std::memcpy(out.data.data() + target, bytes, byteCount);
			    return true;
		    });
		return ok && out.ok;
	}

	bool MatHdf5::StreamDataset(const Dataset& dataset, const ByteSink& sink) const
	{
		if (!dataset.ok || !sink)
		{
			return false;
		}
		const std::size_t elementSize = dataset.type.size;
		if (elementSize == 0)
		{
			return false;
		}

		// 紧凑布局：数据就在消息里（很小）。
		if (dataset.layout == LayoutClass::Compact)
		{
			return sink(dataset.compact.data(), dataset.compact.size(), 0);
		}

		// 连续布局：按 4MB 切片读出（随机读 + 复用缓冲，不产生整文件大小的额外内存）。
		if (dataset.layout == LayoutClass::Contiguous)
		{
			const std::uint64_t total = dataset.ElementCount();
			const std::uint64_t totalBytes = total * elementSize;
			const std::uint64_t available =
			    std::min<std::uint64_t>(dataset.dataBytes,
			                            (dataset.dataAddress < m_size) ? (m_size - dataset.dataAddress)
			                                                           : 0);
			const std::uint64_t bytes = std::min(totalBytes, available);
			// ⚠️ 元数据声明了 N 个元素、文件里却不够：明确失败，而不是静默截短
			//（静默截短会让曲线“看起来只有一半”，很难诊断）。
			if (bytes < totalBytes)
			{
				return false;
			}
			std::size_t slice = kStreamSliceBytes - (kStreamSliceBytes % elementSize);
			if (slice == 0)
			{
				slice = elementSize;
			}
			if (m_slice.size() < slice)
			{
				m_slice.resize(slice);
			}
			for (std::uint64_t offset = 0; offset < bytes; offset += slice)
			{
				const std::size_t count =
				    static_cast<std::size_t>(std::min<std::uint64_t>(slice, bytes - offset));
				if (!ReadAt(dataset.dataAddress + offset, m_slice.data(), count))
				{
					return false;
				}
				if (m_readProgress && !m_readProgress(m_bytesRead, m_size))
				{
					m_readCancelled = true;
					return false;
				}
				if (!sink(m_slice.data(), count, offset / elementSize))
				{
					return false;
				}
			}
			return true;
		}

		// 分块布局：逐 chunk 解压，立刻把每一行交给回调（不保留整块字节数组）。
		if (dataset.chunkAddress == 0 || dataset.chunkDims.empty() || dataset.dims.empty())
		{
			return false;
		}
		// ⚠️ MATLAB 写出的分块维度可能与数据空间维度**维数不一致**（实测：1 维数据集的
		// 分块维度是 `[N, 1]`）。把维度数组在**右侧补 1**（行优先的最快轴）对齐即可。
		std::vector<std::size_t> dims = dataset.dims;
		std::vector<std::size_t> chunkDims = dataset.chunkDims;
		while (dims.size() < chunkDims.size())
		{
			dims.push_back(1);
		}
		while (chunkDims.size() < dims.size())
		{
			chunkDims.push_back(1);
		}
		const std::size_t rank = dims.size();
		std::size_t expected = elementSize;
		for (const std::size_t dim : dataset.chunkDims)
		{
			expected *= dim;
		}
		const std::size_t rowBytes = chunkDims[rank - 1] * elementSize;
		std::size_t rowsPerChunk = 1;
		for (std::size_t axis = 0; axis + 1 < rank; ++axis)
		{
			rowsPerChunk *= chunkDims[axis];
		}

		std::vector<std::uint8_t> inflated;
		std::vector<std::uint64_t> stack{dataset.chunkAddress};
		int guard = 0;
		while (!stack.empty() && guard++ < 1000000)
		{
			const std::uint64_t node = stack.back();
			stack.pop_back();
			if (node == 0 || node + 24 > m_size || !Tag(node, "TREE"))
			{
				continue;
			}
			const std::uint8_t level = U8(node + 5);
			const std::uint16_t entries = U16(node + 6);
			const std::size_t keySize = 8 + 8 * rank;
			std::uint64_t offset = node + 8 + 8 + 8;
			for (std::uint16_t index = 0; index < static_cast<std::uint16_t>(entries + 1u); ++index)
			{
				const std::uint32_t chunkSize = (offset + 4 <= m_size) ? U32(offset) : 0;
				std::vector<std::uint64_t> chunkOffset(rank, 0);
				for (std::size_t axis = 0; axis < rank; ++axis)
				{
					chunkOffset[axis] = U64(offset + 8 + axis * 8);
				}
				offset += keySize;
				if (index >= entries)
				{
					continue;
				}
				const std::uint64_t child = Resolve(U64(offset));
				offset += 8;
				if (level > 0)
				{
					stack.push_back(child);
					continue;
				}
				if (child == 0 || chunkSize == 0 || child + chunkSize > m_size)
				{
					continue;
				}

				// 顺序预读：分块数据通常连续存放，逐 chunk 读会退化成大量小系统调用。
				const std::uint8_t* source = nullptr;
				if (chunkSize <= kReadAheadBytes)
				{
					source = ReadData(child, chunkSize);
				}
				if (source == nullptr)
				{
					if (m_chunk.size() < chunkSize)
					{
						m_chunk.resize(chunkSize);
					}
					if (!ReadAt(child, m_chunk.data(), chunkSize))
					{
						continue;
					}
					if (m_readProgress && !m_readProgress(m_bytesRead, m_size))
					{
						m_readCancelled = true;
						return false;
					}
					source = m_chunk.data();
				}
				if (m_readCancelled)
				{
					return false;
				}
				std::size_t sourceSize = chunkSize;
				if (dataset.deflate && chunkSize != expected)
				{
					// 分块未压缩时大小 = 分块元素数 × 元素大小：此时不需要解压。
					if (!InflateZlib(source, sourceSize, inflated, expected))
					{
						continue;
					}
					if (inflated.size() < expected)
					{
						inflated.resize(expected, 0);
					}
					source = inflated.data();
					sourceSize = expected;
				}

				// 块内按行（最快轴一整行连续）散射到目标位置。
				// ⚠️ 行数可能很大（例如 SDI 的 `#data#` 是 `dims=[N]`、chunk 为 `[N,1]`，
				// 归一化后 rank=2 ⇒「每行只有 1 字节」）。若逐行回调，就是上千万次
				// 小 memcpy + 上千万次 vector 分配，实测把解压吞吐拖到 ~11MB/s。
				// 因此这里把**目标连续的行合并成一次回调**（典型情况整块只需 1 次）。
				//
				// 快路径（rank == 2 且第二维完全对齐）：整块在目标里就是一段连续区域，
				// 直接一次回调，省掉「每行算坐标 + 判断合并」的循环。
				// 实测 BIG_MAT.mat：`#data#` 为 129,139,768 字节 × 32KB chunk，每 chunk 32768 行，
				// 逐行循环共 12.9 亿次迭代（约占总加载时间的一半以上）。
				if (rank == 2 && chunkDims[1] == dims[1] && chunkDims[1] > 0)
				{
					const std::uint64_t startRow =
					    std::min<std::uint64_t>(chunkOffset[0], dims[0]);
					const std::uint64_t rowCount =
					    std::min<std::uint64_t>(chunkDims[0], dims[0] - startRow);
					const std::uint64_t byteCount = rowCount * rowBytes;
					if (startRow < dims[0] && byteCount <= static_cast<std::uint64_t>(sourceSize))
					{
						if (!sink(source, static_cast<std::size_t>(byteCount),
						          startRow * chunkDims[1]))
						{
							return false;
						}
					}
					continue;
				}

				std::vector<std::size_t> coordinate(rank > 0 ? rank - 1 : 0, 0);
				std::uint64_t runElement = 0;		// 合并段的起始元素下标
				std::size_t runSourceOffset = 0;	// 合并段的源偏移
				std::size_t runBytes = 0;			// 合并段字节数
				bool runOpen = false;
				for (std::size_t row = 0; row < rowsPerChunk; ++row)
				{
					std::size_t remainder = row;
					for (std::size_t axis = rank - 1; axis-- > 0;)
					{
						coordinate[axis] = remainder % chunkDims[axis];
						remainder /= chunkDims[axis];
					}
					std::uint64_t targetRow = 0;
					bool inside = true;
					std::uint64_t stride = 1;
					for (std::size_t axis = rank - 1; axis-- > 0;)
					{
						const std::uint64_t index = chunkOffset[axis] + coordinate[axis];
						if (index >= dims[axis])
						{
							inside = false;
							break;
						}
						targetRow += index * stride;
						stride *= dims[axis];
					}
					if (!inside)
					{
						continue;
					}
					const std::size_t sourceOffset = row * rowBytes;
					if (sourceOffset + rowBytes > sourceSize)
					{
						continue;
					}
					const std::uint64_t targetElement = targetRow * chunkDims[rank - 1];
					if (runOpen && targetElement == runElement + (runBytes / elementSize))
					{
						runBytes += rowBytes;
						continue;
					}
					if (runOpen)
					{
						if (!sink(source + runSourceOffset, runBytes, runElement))
						{
							return false;
						}
					}
					runOpen = true;
					runElement = targetElement;
					runSourceOffset = sourceOffset;
					runBytes = rowBytes;
				}
				if (runOpen)
				{
					if (!sink(source + runSourceOffset, runBytes, runElement))
					{
						return false;
					}
				}
			}
		}
		return true;
	}


	std::string MatHdf5::AttributeString(std::uint64_t address, const char* name) const
	{
		std::vector<Message> messages;
		ReadMessages(address, messages);
		for (const Message& message : messages)
		{
			if (message.type != kMessageAttribute || message.data.size() < 8)
			{
				continue;
			}
			const std::uint8_t version = message.data[0];
			const std::size_t headerSize = (version == 1) ? 8 : (version == 2 ? 8 : 9);
			const std::uint16_t nameSize = static_cast<std::uint16_t>(message.data[2]) |
			                               (static_cast<std::uint16_t>(message.data[3]) << 8);
			const std::uint16_t typeSize = static_cast<std::uint16_t>(message.data[4]) |
			                               (static_cast<std::uint16_t>(message.data[5]) << 8);
			const std::uint16_t spaceSize = static_cast<std::uint16_t>(message.data[6]) |
			                                (static_cast<std::uint16_t>(message.data[7]) << 8);
			std::size_t offset = headerSize;
			if (offset + nameSize > message.data.size())
			{
				continue;
			}
			std::string attributeName(reinterpret_cast<const char*>(message.data.data() + offset),
			                          nameSize);
			while (!attributeName.empty() && attributeName.back() == '\0')
			{
				attributeName.pop_back();
			}
			offset += (nameSize + 7) & ~static_cast<std::size_t>(7);
			const std::size_t typeOffset = offset;
			offset += (typeSize + 7) & ~static_cast<std::size_t>(7);
			const std::size_t spaceOffset = offset;
			offset += (spaceSize + 7) & ~static_cast<std::size_t>(7);
			if (attributeName != name)
			{
				continue;
			}
			if (offset > message.data.size())
			{
				return std::string();
			}
			Type type;
			std::size_t consumed = 0;
			if (!ParseType(message.data.data(), message.data.size(), typeOffset, type, consumed))
			{
				return std::string();
			}
			std::size_t count = 1;
			if (spaceOffset + 4 <= message.data.size())
			{
				const std::uint8_t spaceVersion = message.data[spaceOffset];
				const std::uint8_t rank = message.data[spaceOffset + 1];
				std::size_t dimOffset = spaceOffset + ((spaceVersion == 1) ? 8 : 4);
				count = 1;
				for (std::uint8_t axis = 0; axis < rank && dimOffset + 8 <= message.data.size(); ++axis)
				{
					count *= static_cast<std::size_t>(ReadLE(&message.data[dimOffset], 8));
					dimOffset += 8;
				}
			}
			std::string text;
			if (type.cls == TypeClass::String || type.cls == TypeClass::Integer)
			{
				text.assign(reinterpret_cast<const char*>(message.data.data() + offset),
				            std::min(message.data.size() - offset, std::max<std::size_t>(count, 1)));
				while (!text.empty() && text.back() == '\0')
				{
					text.pop_back();
				}
			}
			return text;
		}
		return std::string();
	}

	bool MatHdf5::ReadText(std::uint64_t address, std::wstring& out) const
	{
		out.clear();
		Dataset dataset;
		if (!ReadDataset(address, dataset))
		{
			return false;
		}
		if (dataset.type.cls == TypeClass::String)
		{
			out.assign(dataset.data.begin(), dataset.data.end());
			while (!out.empty() && out.back() == L'\0')
			{
				out.pop_back();
			}
			return true;
		}
		if (dataset.type.size == 2)
		{
			for (std::size_t index = 0; index + 1 < dataset.data.size(); index += 2)
			{
				const wchar_t character = static_cast<wchar_t>(
				    static_cast<std::uint16_t>(dataset.data[index]) |
				    (static_cast<std::uint16_t>(dataset.data[index + 1]) << 8));
				if (character != 0)
				{
					out.push_back(character);
				}
			}
			return true;
		}
		if (dataset.type.size == 1)
		{
			for (const std::uint8_t byte : dataset.data)
			{
				if (byte != 0)
				{
					out.push_back(static_cast<wchar_t>(byte));
				}
			}
			return true;
		}
		return false;
	}

	bool MatHdf5::ReadNumbers(std::uint64_t address, std::vector<double>& out) const
	{
		out.clear();
		Dataset dataset;
		if (!ReadDatasetInfo(address, dataset) || !dataset.ok)
		{
			return false;
		}
		const std::size_t elementSize = dataset.type.size;
		const bool integer = (dataset.type.cls == TypeClass::Integer);
		const bool floating = (dataset.type.cls == TypeClass::Float);
		if (elementSize == 0 || (!integer && !floating))
		{
			return false;
		}
		const std::uint64_t total = dataset.ElementCount();
		if (total == 0 || total > static_cast<std::uint64_t>(SIZE_MAX) / elementSize)
		{
			return false;
		}
		out.assign(static_cast<std::size_t>(total), 0.0);

		// ⚠️ 流式转换：分块数据集逐 chunk 解压后**直接**写进 `out`，
		// 不再先生成一份“整个数据集的未压缩字节”（GB 级数据下那份缓冲与 double 输出同样大）。
		return StreamDataset(
		    dataset, [&out, elementSize, integer, floating, &dataset](const std::uint8_t* bytes,
		                                                              std::size_t byteCount,
		                                                              std::uint64_t elementOffset) -> bool
		    {
			    if (elementSize == 0 || byteCount % elementSize != 0)
			    {
				    return false;
			    }
			    const std::size_t count = byteCount / elementSize;
			    if (elementOffset + count > out.size())
			    {
				    return false;
			    }
			    ConvertNumbers(bytes, count, elementSize, integer, floating, dataset.type.isSigned,
			                   out.data() + elementOffset);
			    return true;
		    });
	}

	bool MatHdf5::ReadDoubleStream(std::uint64_t address, std::vector<double>& out) const
	{
		out.clear();
		Dataset dataset;
		if (!ReadDatasetInfo(address, dataset) || !dataset.ok)
		{
			return false;
		}
		// SDI 的 `#data#` 必须是单字节元素（uint8 字节流），每 8 字节一组是一个 double。
		if (dataset.type.size != 1)
		{
			return false;
		}
		const std::uint64_t totalBytes = dataset.ElementCount();
		if (totalBytes == 0 || totalBytes % sizeof(double) != 0)
		{
			return false;
		}
		const std::size_t sampleCount = static_cast<std::size_t>(totalBytes / sizeof(double));
		out.assign(sampleCount, 0.0);

		// ⚠️ 分块边界不一定与 8 字节对齐（chunk 维度是任意整数）⇒ 用最多 7 字节的尾巴接续。
		// 关键：**不做整块字节缓冲**，解压出来的字节立即被解释成 double 写进 `out`。
		std::uint8_t tail[8] = {};
		std::size_t tailBytes = 0;
		std::uint64_t tailStart = 0;	// 尾巴首字节在数据集中的字节偏移
		bool failed = false;
		const bool ok = StreamDataset(
		    dataset, [&](const std::uint8_t* bytes, std::size_t byteCount,
		                 std::uint64_t elementOffset) -> bool
		    {
			    // 数据必须连续吐出（分块遍历按坐标升序；中间有空洞说明表里缺块）。
			    if (tailBytes > 0 && elementOffset != tailStart + tailBytes)
			    {
				    failed = true;
				    return false;
			    }
			    std::size_t position = 0;
			    if (tailBytes == 0)
			    {
				    tailStart = elementOffset;
			    }
			    // 先把尾巴补满一个 double。
			    while (tailBytes > 0 && tailBytes < sizeof(double) && position < byteCount)
			    {
				    tail[tailBytes++] = bytes[position++];
			    }
			    if (tailBytes == sizeof(double))
			    {
				    const std::size_t index = static_cast<std::size_t>(tailStart / sizeof(double));
				    if (index >= out.size())
				    {
					    failed = true;
					    return false;
				    }
				    std::memcpy(&out[index], tail, sizeof(double));
				    tailStart += sizeof(double);
				    tailBytes = 0;
			    }
			    // 剩余部分按 8 字节整组直接 memcpy（容量已按总字节数算好）。
			    const std::size_t groups = (byteCount - position) / sizeof(double);
			    if (groups > 0)
			    {
				    const std::size_t index = static_cast<std::size_t>(tailStart / sizeof(double));
				    if (index + groups > out.size())
				    {
					    failed = true;
					    return false;
				    }
				    std::memcpy(out.data() + index, bytes + position, groups * sizeof(double));
				    position += groups * sizeof(double);
				    tailStart += groups * sizeof(double);
			    }
			    // 末尾零头留作下一块的尾巴。
			    while (position < byteCount)
			    {
				    tail[tailBytes++] = bytes[position++];
			    }
			    return true;
		    });
		if (!ok || failed)
		{
			out.clear();
			return false;
		}
		// 尾部不足 8 字节（损坏）视为失败，与旧实现（要求整除）保持一致。
		if (tailBytes != 0)
		{
			out.clear();
			return false;
		}
		return true;
	}

	bool MatHdf5::ReadReferences(std::uint64_t address, std::vector<std::uint64_t>& out) const
	{
		out.clear();
		Dataset dataset;		if (!ReadDataset(address, dataset) || !dataset.ok ||
		    dataset.type.cls != TypeClass::Reference)
		{
			return false;
		}
		const std::size_t count = dataset.data.size() / 8;
		out.reserve(count);
		for (std::size_t index = 0; index < count; ++index)
		{
			const std::uint64_t raw = ReadLE(dataset.data.data() + index * 8, 8);
			out.push_back(Resolve(raw));
		}
		return true;
	}

} // namespace jdi