Index: mptrack/Mptrack.h =================================================================== --- mptrack/Mptrack.h (revision 24610) +++ mptrack/Mptrack.h (working copy) @@ -35,6 +35,7 @@ class TrackerSettings; class ImmediateWindowsIniFileSettingsBackend; class BatchedWindowsIniFileSettingsBackend; +class CachedIniFileSettingsBackend; template class FileSettingsContainer; #ifndef IniFileSettingsBackend #define IniFileSettingsBackend ImmediateWindowsIniFileSettingsBackend Index: mptrack/SettingsIni.cpp =================================================================== --- mptrack/SettingsIni.cpp (revision 24610) +++ mptrack/SettingsIni.cpp (working copy) @@ -13,25 +13,44 @@ #include "SettingsIni.h" +#include "mpt/base/integer.hpp" #include "mpt/binary/hex.hpp" #include "mpt/io_file/fstream.hpp" #include "mpt/io_file/outputfile.hpp" #include "mpt/parse/parse.hpp" +#include "mpt/base/bit.hpp" +#include "mpt/io_read/filecursor.hpp" +#include "mpt/io_read/filecursor_memory.hpp" +#include "mpt/io_read/filereader.hpp" +#include "mpt/format/message_macros.hpp" +#include "mpt/string/types.hpp" +#include "mpt/string_transcode/transcode.hpp" + #include "../common/misc_util.h" #include "../common/mptFileIO.h" #include "../common/mptStringBuffer.h" #include +#include +#include +#include +#include +#include +#include #include +#include +#include + + OPENMPT_NAMESPACE_BEGIN -WindowsIniFileBase::WindowsIniFileBase(mpt::PathString filename_) +IniFileBase::IniFileBase(mpt::PathString filename_) : filename(std::move(filename_)) , file(filename) { @@ -38,6 +57,24 @@ return; } +const mpt::PathString &IniFileBase::Filename() const +{ + return filename; +} + +mpt::PathString IniFileBase::GetFilename() const +{ + return filename; +} + + + +WindowsIniFileBase::WindowsIniFileBase(mpt::PathString filename_) + : IniFileBase(std::move(filename_)) +{ + return; +} + void WindowsIniFileBase::ConvertToUnicode(const mpt::ustring &backupTag) { // Force ini file to be encoded in UTF16. @@ -80,18 +117,111 @@ #endif } -const mpt::PathString &WindowsIniFileBase::Filename() const + + +IniProbeResult IniFileHelpers::Probe(mpt::const_byte_span data) { - return filename; + IniProbeResult result; + const std::array bom_utf32be = {mpt::byte_cast(uint8{0x00}), mpt::byte_cast(uint8{0x00}), mpt::byte_cast(uint8{0xfe}), mpt::byte_cast(uint8{0xff})}; + const std::array bom_utf32le = {mpt::byte_cast(uint8{0xff}), mpt::byte_cast(uint8{0xfe}), mpt::byte_cast(uint8{0x00}), mpt::byte_cast(uint8{0x00})}; + const std::array bom_utf16be = {mpt::byte_cast(uint8{0xfe}), mpt::byte_cast(uint8{0xff})}; + const std::array bom_utf16le = {mpt::byte_cast(uint8{0xff}), mpt::byte_cast(uint8{0xfe})}; + const std::array bom_utf8 = {mpt::byte_cast(uint8{0xEF}), mpt::byte_cast(uint8{0xBB}), mpt::byte_cast(uint8{0xBF})}; + if((data.size() >= bom_utf32be.size()) && (std::memcmp(data.data(), bom_utf32be.data(), bom_utf32be.size()) == 0)) + { + result.encoding = IniEncoding::UTF32BE; + result.data_offset = 4; + } else if((data.size() >= bom_utf32le.size()) && (std::memcmp(data.data(), bom_utf32le.data(), bom_utf32le.size()) == 0)) + { + result.encoding = IniEncoding::UTF32LE; + result.data_offset = 4; + } else if((data.size() >= bom_utf16be.size()) && (std::memcmp(data.data(), bom_utf16be.data(), bom_utf16be.size()) == 0)) + { + result.encoding = IniEncoding::UTF16BE; + result.data_offset = 2; + } else if((data.size() >= bom_utf16le.size()) && (std::memcmp(data.data(), bom_utf16le.data(), bom_utf16le.size()) == 0)) + { + result.encoding = IniEncoding::UTF16LE; + result.data_offset = 2; + } else if((data.size() >= bom_utf8.size()) && (std::memcmp(data.data(), bom_utf8.data(), bom_utf8.size()) == 0)) + { + result.encoding = IniEncoding::UTF8; + result.data_offset = 3; + } else + { + result.encoding = IniEncoding::ANSI; + result.data_offset = 0; + } + + return result; } -mpt::PathString WindowsIniFileBase::GetFilename() const +static mpt::ustring IniEncodingToString(IniEncoding encoding) { - return filename; + switch(encoding) + { + case IniEncoding::UTF32BE: + return U_("UTF-32BE"); + break; + case IniEncoding::UTF32LE: + return U_("UTF-32LE"); + break; + case IniEncoding::UTF16BE: + return U_("UTF-16BE"); + break; + case IniEncoding::UTF16LE: + return U_("UTF-16LE"); + break; + case IniEncoding::UTF8: + return U_("UTF-8"); + break; + case IniEncoding::ANSI: + return U_("ANSI"); + break; + } } +std::list> IniFileHelpers::CreateIniHeader(IniVersion version, IniEncoding encoding) +{ + std::list> result; + switch(version.major) + { + case 1: + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!Format")}, U_("com.microsoft.fileformat.ini"))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!Encoding")}, U_("ANSI"))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!CaseSensitive")}, static_cast(0))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionMajor")}, static_cast(version.major))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionMinor")}, static_cast(version.minor))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionPatch")}, static_cast(version.patch))); + break; + case 2: + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!Format")}, U_("com.microsoft.fileformat.ini"))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!Encoding")}, U_("UTF-16LE"))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!CaseSensitive")}, static_cast(0))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionMajor")}, static_cast(version.major))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionMinor")}, static_cast(version.minor))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionPatch")}, static_cast(version.patch))); + break; + case 3: + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!Format")}, U_("org.winehq.fileformat.ini"))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!Encoding")}, U_("UTF-8"))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!CaseSensitive")}, static_cast(0))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionMajor")}, static_cast(version.major))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionMinor")}, static_cast(version.minor))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionPatch")}, static_cast(version.patch))); + break; + case 4: + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!Format")}, U_("org.openmpt.fileformat.ini"))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!Encoding")}, IniEncodingToString(encoding))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!CaseSensitive")}, static_cast(1))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionMajor")}, static_cast(version.major))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionMinor")}, static_cast(version.minor))); + result.emplace_back(std::make_pair(SettingPath{U_("!Type"), U_("!VersionPatch")}, static_cast(version.patch))); + break; + } + return result; +} - mpt::winstring IniFileHelpers::GetSection(const SettingPath &path) { return mpt::ToWin(path.GetSection()); @@ -558,4 +688,724 @@ +static inline bool is_hex(mpt::uchar c) +{ + return (MPT_UCHAR('0') <= c && c <= MPT_UCHAR('9')) || (MPT_UCHAR('a') <= c && c <= MPT_UCHAR('f')) || (MPT_UCHAR('A') <= c && c <= MPT_UCHAR('F')); +} + +static mpt::ustring escape(mpt::ustring_view text) +{ + mpt::ustring result; + const std::size_t len = text.length(); + result.reserve(len); + for(std::size_t pos = 0; pos < len; ++pos) + { + const mpt::uchar c = text[pos]; + if((c == MPT_UCHAR('^')) || (c == MPT_UCHAR(';')) || (c == MPT_UCHAR('[')) || (c == MPT_UCHAR(']')) || (c == MPT_UCHAR('=')) || (c == MPT_UCHAR('\"'))) + { + result.push_back(MPT_UCHAR('^')); + result.push_back(c); + } else if(mpt::char_value(c) < 32) + { + result.push_back(MPT_UCHAR('^')); + result.push_back(MPT_UCHAR('x')); + result.append(mpt::ufmt::hex0<2>(mpt::char_value(c))); + } else if((0x80 <= mpt::char_value(c)) && (mpt::char_value(c) <= 0x9f)) + { + result.push_back(MPT_UCHAR('^')); + result.push_back(MPT_UCHAR('x')); + result.append(mpt::ufmt::hex0<2>(mpt::char_value(c))); + } else + { + result.push_back(c); + } + } + return result; +} + +static mpt::ustring unescape(mpt::ustring_view text) +{ + mpt::ustring result; + const std::size_t len = text.length(); + result.reserve(len); + for(std::size_t pos = 0; pos < len; ++pos) + { + std::size_t rem = len - pos; + if(text[pos] == MPT_UCHAR('^')) + { + if((rem >= 2) && (text[pos+1] == MPT_UCHAR('^'))) + { + result.push_back(MPT_UCHAR('^')); + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR(';'))) + { + result.push_back(MPT_UCHAR(';')); + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR('['))) + { + result.push_back(MPT_UCHAR('[')); + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR(']'))) + { + result.push_back(MPT_UCHAR(']')); + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR('='))) + { + result.push_back(MPT_UCHAR('=')); + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR('\"'))) + { + result.push_back(MPT_UCHAR('\"')); + pos += 1; + } else if((rem >= 4) && (text[pos+1] == MPT_UCHAR('x')) && is_hex(text[pos+2]) && is_hex(text[pos+3])) + { + result.push_back(static_cast(mpt::parse_hex(mpt::ustring{text.substr(pos + 2, 2)}))); + pos += 3; + } else if((rem >= 6) && (text[pos+1] == MPT_UCHAR('u')) && is_hex(text[pos+2]) && is_hex(text[pos+3]) && is_hex(text[pos+4]) && is_hex(text[pos+5])) + { + result.append(mpt::transcode(std::u16string(1, static_cast(mpt::parse_hex(mpt::ustring{text.substr(pos + 2, 4)}))))); + pos += 5; + } else if((rem >= 10) && (text[pos+1] == MPT_UCHAR('U')) && is_hex(text[pos+2]) && is_hex(text[pos+3]) && is_hex(text[pos+4]) && is_hex(text[pos+5]) && is_hex(text[pos+6]) && is_hex(text[pos+7]) && is_hex(text[pos+8]) && is_hex(text[pos+9])) + { + result.append(mpt::transcode(std::u32string(1, static_cast(mpt::parse_hex(mpt::ustring{text.substr(pos + 2, 8)}))))); + pos += 9; + } else + { + result.push_back(text[pos]); + } + } else + { + result.push_back(text[pos]); + } + } + return result; +} + +static inline std::size_t find_unescaped_first(mpt::ustring_view text, mpt::uchar c) +{ + const std::size_t len = text.length(); + for(std::size_t pos = 0; pos < len; ++pos) + { + if(text[pos] == c) + { + return pos; + } + std::size_t rem = len - pos; + if(text[pos] == MPT_UCHAR('^')) + { + if((rem >= 2) && (text[pos+1] == MPT_UCHAR('^'))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR(';'))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR('['))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR(']'))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR('='))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR('\"'))) + { + pos += 1; + } else if((rem >= 4) && (text[pos+1] == MPT_UCHAR('x')) && is_hex(text[pos+2]) && is_hex(text[pos+3])) + { + pos += 3; + } else if((rem >= 6) && (text[pos+1] == MPT_UCHAR('u')) && is_hex(text[pos+2]) && is_hex(text[pos+3]) && is_hex(text[pos+4]) && is_hex(text[pos+5])) + { + pos += 5; + } else if((rem >= 10) && (text[pos+1] == MPT_UCHAR('U')) && is_hex(text[pos+2]) && is_hex(text[pos+3]) && is_hex(text[pos+4]) && is_hex(text[pos+5]) && is_hex(text[pos+6]) && is_hex(text[pos+7]) && is_hex(text[pos+8]) && is_hex(text[pos+9])) + { + pos += 9; + } + } + } + return mpt::ustring_view::npos; +} + +static inline std::size_t find_unescaped_last(mpt::ustring_view text, mpt::uchar c) +{ + std::size_t result = mpt::ustring_view::npos; + const std::size_t len = text.length(); + for(std::size_t pos = 0; pos < len; ++pos) + { + if(text[pos] == c) + { + result = pos; + } + std::size_t rem = len - pos; + if(text[pos] == MPT_UCHAR('^')) + { + if((rem >= 2) && (text[pos+1] == MPT_UCHAR('^'))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR(';'))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR('['))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR(']'))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR('='))) + { + pos += 1; + } else if((rem >= 2) && (text[pos+1] == MPT_UCHAR('\"'))) + { + pos += 1; + } else if((rem >= 4) && (text[pos+1] == MPT_UCHAR('x')) && is_hex(text[pos+2]) && is_hex(text[pos+3])) + { + pos += 3; + } else if((rem >= 6) && (text[pos+1] == MPT_UCHAR('u')) && is_hex(text[pos+2]) && is_hex(text[pos+3]) && is_hex(text[pos+4]) && is_hex(text[pos+5])) + { + pos += 5; + } else if((rem >= 10) && (text[pos+1] == MPT_UCHAR('U')) && is_hex(text[pos+2]) && is_hex(text[pos+3]) && is_hex(text[pos+4]) && is_hex(text[pos+5]) && is_hex(text[pos+6]) && is_hex(text[pos+7]) && is_hex(text[pos+8]) && is_hex(text[pos+9])) + { + pos += 9; + } + } + } + return result; +} + +static std::vector split_any_line_ending(mpt::ustring_view text) +{ + std::vector result; + const std::size_t len = text.length(); + std::size_t beg = 0; + for(std::size_t pos = 0; pos < len; ++pos) + { + std::size_t rem = len - pos; + if((rem >= 2) && (text[pos+0] == MPT_UCHAR('\r')) && (text[pos+1] == MPT_UCHAR('\n'))) + { + result.emplace_back(text.substr(beg, pos - beg)); + pos += 2; + beg = pos; + } else if((rem >=1) && text[pos+0] == MPT_UCHAR('\n')) + { + result.emplace_back(text.substr(beg, pos - beg)); + pos += 1; + beg = pos; + } else if((rem >=1) && text[pos+0] == MPT_UCHAR('\r')) + { + result.emplace_back(text.substr(beg, pos - beg)); + pos += 1; + beg = pos; + } + } + if(!result.empty() || ((len - beg) > 0)) + { + result.emplace_back(text.substr(beg, len - beg)); + } + return result; +} + +static inline mpt::ustring_view trim_whitespace(mpt::ustring_view text) +{ + mpt::ustring_view::size_type beg_pos = text.find_first_not_of(MPT_ULITERAL(" \t\r\n")); + if(beg_pos != mpt::ustring_view::npos) + { + text = text.substr(beg_pos); + } else + { + text = text.substr(text.length(), 0); + } + mpt::ustring_view::size_type end_pos = text.find_last_not_of(MPT_ULITERAL(" \t\r\n")); + if(end_pos != mpt::ustring_view::npos) + { + text = text.substr(0, end_pos + 1); + } else + { + text = text.substr(0, 0); + } + return text; +} + +static inline mpt::ustring_view remove_quotes(mpt::ustring_view text) +{ + if(text.length() <= 1) + { + return text; + } + mpt::ustring_view::size_type beg_quote = find_unescaped_first(text, MPT_UCHAR('\"')); + mpt::ustring_view::size_type end_quote = find_unescaped_last(text, MPT_UCHAR('\"')); + if((beg_quote == 0) && (end_quote == (text.length() - 1))) + { + return text.substr(1, text.length() - 2); + } + return text; +} + +void CachedIniFileSettingsBackend::ReadFileIntoCache() +{ + cache.clear(); + const std::vector filedataraw = file.read(); + IniProbeResult probe = Probe(mpt::as_span(filedataraw)); + const mpt::span filedata = mpt::as_span(filedataraw).subspan(probe.data_offset); + mpt::ustring filetext; + switch(probe.encoding) + { + case IniEncoding::UTF32BE: + { + if((filedata.size() % sizeof(char32_t)) != 0) + { + MPT_LOG_GLOBAL(LogError, "Settings", MPT_UFORMAT_MESSAGE("{}: {}")(filename, U_("UTF32 encoding detected, but file size is not a multiple of 4."))); + } + MPT_MAYBE_CONSTANT_IF(mpt::endian_is_big()) + { + std::u32string utf32data; + std::size_t count = filedata.size() / sizeof(char32_t); + utf32data.resize(count); + std::memcpy(utf32data.data(), filedata.data(), count * sizeof(char32_t)); + filetext = mpt::transcode(utf32data); + } else + { + auto fc = mpt::IO::make_FileCursor(filedata); + std::u32string utf32data; + utf32data.reserve(fc.GetLength() / sizeof(char32_t)); + while(!fc.EndOfFile()) + { + utf32data.push_back(static_cast(mpt::IO::FileReader::ReadInt32BE(fc))); + } + filetext = mpt::transcode(utf32data); + } + } + break; + case IniEncoding::UTF32LE: + { + if((filedata.size() % sizeof(char32_t)) != 0) + { + MPT_LOG_GLOBAL(LogError, "Settings", MPT_UFORMAT_MESSAGE("{}: {}")(filename, U_("UTF32 encoding detected, but file size is not a multiple of 4."))); + } + MPT_MAYBE_CONSTANT_IF(mpt::endian_is_little()) + { + std::u32string utf32data; + std::size_t count = filedata.size() / sizeof(char32_t); + utf32data.resize(count); + std::memcpy(utf32data.data(), filedata.data(), count * sizeof(char32_t)); + filetext = mpt::transcode(utf32data); + } else + { + auto fc = mpt::IO::make_FileCursor(filedata); + std::u32string utf32data; + utf32data.reserve(fc.GetLength() / sizeof(char32_t)); + while(!fc.EndOfFile()) + { + utf32data.push_back(static_cast(mpt::IO::FileReader::ReadInt32LE(fc))); + } + filetext = mpt::transcode(utf32data); + } + } + break; + case IniEncoding::UTF16BE: + { + if((filedata.size() % sizeof(char16_t)) != 0) + { + MPT_LOG_GLOBAL(LogError, "Settings", MPT_UFORMAT_MESSAGE("{}: {}")(filename, U_("UTF16 encoding detected, but file size is not a multiple of 2."))); + } + MPT_MAYBE_CONSTANT_IF(mpt::endian_is_big()) + { + std::u16string utf16data; + std::size_t count = filedata.size() / sizeof(char16_t); + utf16data.resize(count); + std::memcpy(utf16data.data(), filedata.data(), count * sizeof(char16_t)); + filetext = mpt::transcode(utf16data); + } else + { + auto fc = mpt::IO::make_FileCursor(filedata); + std::u16string utf16data; + utf16data.reserve(fc.GetLength() / sizeof(char16_t)); + while(!fc.EndOfFile()) + { + utf16data.push_back(static_cast(mpt::IO::FileReader::ReadInt16BE(fc))); + } + filetext = mpt::transcode(utf16data); + } + } + break; + case IniEncoding::UTF16LE: + { + if((filedata.size() % sizeof(char16_t)) != 0) + { + MPT_LOG_GLOBAL(LogError, "Settings", MPT_UFORMAT_MESSAGE("{}: {}")(filename, U_("UTF16 encoding detected, but file size is not a multiple of 2."))); + } + MPT_MAYBE_CONSTANT_IF(MPT_OS_WINDOWS) + { + std::wstring utf16data; + std::size_t count = filedata.size() / sizeof(wchar_t); + utf16data.resize(count); + std::memcpy(utf16data.data(), filedata.data(), count * sizeof(wchar_t)); + filetext = mpt::transcode(utf16data); + } else MPT_MAYBE_CONSTANT_IF(mpt::endian_is_little()) + { + std::u16string utf16data; + std::size_t count = filedata.size() / sizeof(char16_t); + utf16data.resize(count); + std::memcpy(utf16data.data(), filedata.data(), count * sizeof(char16_t)); + filetext = mpt::transcode(utf16data); + } else + { + auto fc = mpt::IO::make_FileCursor(filedata); + std::u16string utf16data; + utf16data.reserve(fc.GetLength() / sizeof(char16_t)); + while(!fc.EndOfFile()) + { + utf16data.push_back(static_cast(mpt::IO::FileReader::ReadInt16LE(fc))); + } + filetext = mpt::transcode(utf16data); + } + } + break; + case IniEncoding::UTF8: + filetext = mpt::transcode(mpt::common_encoding::utf8, mpt::buffer_cast(filedata)); + break; + case IniEncoding::ANSI: + filetext = mpt::transcode(mpt::logical_encoding::locale, mpt::buffer_cast(filedata)); + break; + } +#if 0 + mpt::ustring lineending; + switch(probe.lineending) + { + case IniLineEnding::CRLF: + lineending = MPT_USTRING("\r\n"); + break; + case IniLineEnding::LF: + lineending = MPT_USTRING("\n"); + break; + case IniLineEnding::CR: + lineending = MPT_USTRING("\r"); + break; + } + const std::vector lines = mpt::split(filetext, lineending); +#else + const std::vector lines = split_any_line_ending(filetext); +#endif + mpt::ustring current_section; + mpt::ustring last_key; + bool empty_line_after_last_key = false; + std::vector running_comments; + std::size_t line_number = 0; + bool last_line_empty = false; + auto store_previous_comments = [&]() + { + if(!empty_line_after_last_key && (running_comments.size() > 0)) + { + comments[std::make_pair(current_section, last_key)].after = std::move(running_comments); + running_comments = {}; + } + }; + for(const auto &raw_line : lines) + { + line_number += 1; + last_line_empty = (raw_line.length() == 0); + mpt::ustring_view line = trim_whitespace(raw_line); + std::size_t line_len = line.length(); + if(line_len == 0) + { // empty line + store_previous_comments(); + empty_line_after_last_key = true; + continue; + } + if(find_unescaped_first(line, MPT_UCHAR(';')) == 0) + { // comment + running_comments.push_back(mpt::ustring{trim_whitespace(line.substr(1))}); + continue; + } + if(find_unescaped_first(line, MPT_UCHAR('[')) == 0) + { // section start + mpt::ustring_view::size_type opening_pos = 0; + mpt::ustring_view::size_type closing_pos = find_unescaped_first(line, MPT_UCHAR(']')); + if(closing_pos == mpt::ustring_view::npos) + { + store_previous_comments(); + empty_line_after_last_key = true; + MPT_LOG_GLOBAL(LogError, "Settings", MPT_UFORMAT_MESSAGE("{}({}): No closing bracket in section header. ('{}').")(filename, line_number, raw_line)); + continue; + } + mpt::ustring_view remainder = line.substr(closing_pos + 1); + if(trim_whitespace(remainder).length() > 0) + { + MPT_LOG_GLOBAL(LogWarning, "Settings", MPT_UFORMAT_MESSAGE("{}({}): Junk after section header. ('{}').")(filename, line_number, raw_line)); + } + mpt::ustring_view escaped_section = trim_whitespace(line.substr(opening_pos + 1, closing_pos - opening_pos - 1)); + mpt::ustring section = unescape(escaped_section); + if(cache.find(section) != cache.end()) + { + store_previous_comments(); + empty_line_after_last_key = true; + current_section.clear(); + MPT_LOG_GLOBAL(LogWarning, "Settings", MPT_UFORMAT_MESSAGE("{}({}): Ignoring duplicate section '{}'. ('{}').")(filename, line_number, escape(section), raw_line)); + continue; + } + MPT_LOG_GLOBAL(LogInformation, "Settings", MPT_UFORMAT_MESSAGE("{}({}): [{}]")(filename, line_number, escape(section))); + comments[std::make_pair(section, mpt::ustring{})].before = std::move(running_comments); + running_comments = {}; + cache[section].emplace(); + current_section = std::move(section); + last_key.clear(); + empty_line_after_last_key = false; + continue; + } + mpt::ustring_view::size_type equal_pos = find_unescaped_first(line, MPT_UCHAR('=')); + if(equal_pos == mpt::ustring_view::npos) + { // syntax error + store_previous_comments(); + empty_line_after_last_key = true; + MPT_LOG_GLOBAL(LogError, "Settings", MPT_UFORMAT_MESSAGE("{}({}): Syntax error: Invalid token. ('{}').")(filename, line_number, raw_line)); + continue; + } + // key value + if(current_section.empty()) + { + store_previous_comments(); + empty_line_after_last_key = true; + MPT_LOG_GLOBAL(LogWarning, "Settings", MPT_UFORMAT_MESSAGE("{}({}): Ignoring key-value without section. ('{}').")(filename, line_number, raw_line)); + continue; + } + mpt::ustring_view escaped_key = trim_whitespace(line.substr(0, equal_pos)); + mpt::ustring_view escaped_val = remove_quotes(trim_whitespace(line.substr(equal_pos + 1))); + if(escaped_key.empty()) + { + store_previous_comments(); + empty_line_after_last_key = true; + MPT_LOG_GLOBAL(LogError, "Settings", MPT_UFORMAT_MESSAGE("{}({}): Syntax error: Empty key. ('{}').")(filename, line_number, raw_line)); + continue; + } + mpt::ustring key = unescape(escaped_key); + mpt::ustring val = unescape(escaped_val); + if(cache[current_section].value().find(key) != cache[current_section].value().end()) + { + store_previous_comments(); + empty_line_after_last_key = true; + MPT_LOG_GLOBAL(LogWarning, "Settings", MPT_UFORMAT_MESSAGE("{}({}): Ignoring duplicate key '{}'. ('{}').")(filename, line_number, escape(key), raw_line)); + continue; + } + MPT_LOG_GLOBAL(LogInformation, "Settings", MPT_UFORMAT_MESSAGE("{}({}): {}={}")(filename, line_number, escape(key), escape(val))); + comments[std::make_pair(current_section, key)].before = std::move(running_comments); + running_comments = {}; + cache[current_section].value()[key] = std::make_optional(std::move(val)); + last_key = std::move(key); + empty_line_after_last_key = false; + } + store_previous_comments(); + if(!last_line_empty) + { + MPT_LOG_GLOBAL(LogNotification, "Settings", MPT_UFORMAT_MESSAGE("{}({}): {}")(filename, line_number, U_("No newline after last line."))); + } +} + +void CachedIniFileSettingsBackend::MergeSettingsIntoCache(const std::set &removeSections, const std::map> &settings) +{ + for(const auto §ion : removeSections) + { + cache.erase(section); + } + std::map>> sectionssettings; + for(const auto &[path, value] : settings) + { + sectionssettings[path.GetRefSection()][path] = value; + } + for(const auto &[section, sectionsettings] : sectionssettings) + { + if(!cache[section].has_value()) + { + cache[section].emplace(); + } + std::map> &workingsectionsettings = cache[section].value(); + for(const auto &[path, value] : sectionsettings) + { + if(value.has_value()) + { + workingsectionsettings[path.GetRefKey()] = FormatValueAsIni(value.value()); + } else + { + workingsectionsettings.erase(path.GetRefKey()); + } + } + } +} + +void CachedIniFileSettingsBackend::WriteCacheIntoFile() +{ + IniEncoding encoding = IniEncoding::UTF8; +#if MPT_OS_WINDOWS && defined(UNICODE) + if(mpt::OS::Windows::IsWine()) + { + encoding = IniEncoding::UTF8; + } else + { + encoding = IniEncoding::UTF16LE; + } +#else + encoding = IniEncoding::UTF8; +#endif + mpt::ustring filetext; + const std::list> header = CreateIniHeader(IniVersion{4, 0, 0}, encoding); + for(const auto &path : header) + { + cache.erase(path.first.GetRefSection()); + } + const mpt::ustring newline = MPT_USTRING("\r\n"); + mpt::ustring last_section; + for(const auto &[path, value] : header) + { + if(path.GetRefSection() != last_section) + { + last_section = path.GetRefSection(); + filetext += MPT_UFORMAT_MESSAGE("[{}]{}")(escape(path.GetRefSection()), newline); + } + filetext += MPT_UFORMAT_MESSAGE("{}={}{}")(escape(path.GetRefKey()), escape(FormatValueAsIni(value)), newline); + } + for(const auto &[section, sectionsettings] : cache) + { + filetext += newline; + auto sectioncomments = comments.find(std::make_pair(section, mpt::ustring{})); + if(sectioncomments != comments.end()) + { + for(const auto &comment : sectioncomments->second.before) + { + filetext += MPT_UFORMAT_MESSAGE(";{}{}")(comment, newline); + } + } + filetext += MPT_UFORMAT_MESSAGE("[{}]{}")(escape(section), newline); + if(sectioncomments != comments.end()) + { + for(const auto &comment : sectioncomments->second.after) + { + filetext += MPT_UFORMAT_MESSAGE(";{}{}")(comment, newline); + } + if(sectioncomments->second.after.size() > 0) + { + filetext += newline; + } + } + if(sectionsettings.has_value()) + { + for(const auto &[key, value] : sectionsettings.value()) + { + if(value.has_value()) + { + auto keycomments = comments.find(std::make_pair(section, key)); + if(keycomments != comments.end()) + { + for(const auto &comment : keycomments->second.before) + { + filetext += MPT_UFORMAT_MESSAGE(";{}{}")(comment, newline); + } + } + const mpt::ustring escaped_value = escape(value.value()); + if(escaped_value != trim_whitespace(escaped_value)) + { + filetext += MPT_UFORMAT_MESSAGE("{}=\"{}\"{}")(escape(key), escaped_value, newline); + } else + { + filetext += MPT_UFORMAT_MESSAGE("{}={}{}")(escape(key), escaped_value, newline); + } + if(keycomments != comments.end()) + { + for(const auto &comment : keycomments->second.after) + { + filetext += MPT_UFORMAT_MESSAGE(";{}{}")(comment, newline); + } + if(keycomments->second.after.size() > 0) + { + filetext += newline; + } + } + } + } + } + } + [[maybe_unused]] const std::array utf16le_bom = {mpt::byte_cast(uint8{0xff}), mpt::byte_cast(uint8{0xfe})}; + [[maybe_unused]] const std::array utf8_bom = {mpt::byte_cast(uint8{0xef}), mpt::byte_cast(uint8{0xbb}), mpt::byte_cast(uint8{0xbf})}; + std::vector filedata; +#if MPT_OS_WINDOWS && defined(UNICODE) + if(mpt::OS::Windows::IsWine()) + { + MPT_ASSERT(encoding == IniEncoding::UTF8); + mpt::append(filedata, utf8_bom); + mpt::append(filedata, mpt::byte_cast(mpt::as_span(mpt::transcode(mpt::common_encoding::utf8, filetext)))); + } else + { + MPT_ASSERT(encoding == IniEncoding::UTF16LE); + mpt::append(filedata, utf16le_bom); + std::wstring wtext = mpt::transcode(filetext); + mpt::append(filedata, mpt::as_span(reinterpret_cast(wtext.data()), wtext.size() * sizeof(wchar_t))); + } +#else + MPT_ASSERT(encoding == IniEncoding::UTF8); + mpt::append(filedata, utf8_bom); + mpt::append(filedata, mpt::byte_cast(mpt::as_span(mpt::transcode(mpt::common_encoding::utf8, filetext)))); +#endif + file.write(filedata); +} + +void CachedIniFileSettingsBackend::InvalidateCache() +{ + OPENMPT_PROFILE_FUNCTION(Profiler::Settings); + std::lock_guard l(file); + ReadFileIntoCache(); +} + +SettingValue CachedIniFileSettingsBackend::ReadSetting(const SettingPath &path, const SettingValue &def) const +{ + OPENMPT_PROFILE_FUNCTION(Profiler::Settings); + const auto sectionit = cache.find(path.GetRefSection()); + if(sectionit != cache.end()) + { + if(sectionit->second.has_value()) + { + const std::map> §ion = sectionit->second.value(); + const auto it = section.find(path.GetRefKey()); + if(it != section.end()) + { + if(it->second.has_value()) + { + return ParseValueFromIni(it->second.value(), def); + } else + { + return def; + } + } else + { + return def; + } + } else + { + return def; + } + } else + { + return def; + } +} + +void CachedIniFileSettingsBackend::WriteAllSettings(const std::set &removeSections, const std::map> &settings) +{ + OPENMPT_PROFILE_FUNCTION(Profiler::Settings); + std::lock_guard l(file); + ReadFileIntoCache(); + MergeSettingsIntoCache(removeSections, settings); + WriteCacheIntoFile(); +} + +CachedIniFileSettingsBackend::CachedIniFileSettingsBackend(mpt::PathString filename_) + : IniFileBase(std::move(filename_)) +{ + OPENMPT_PROFILE_FUNCTION(Profiler::Settings); + std::lock_guard l(file); + ReadFileIntoCache(); +} + +CachedIniFileSettingsBackend::~CachedIniFileSettingsBackend() +{ + return; +} + + OPENMPT_NAMESPACE_END Index: mptrack/SettingsIni.h =================================================================== --- mptrack/SettingsIni.h (revision 24610) +++ mptrack/SettingsIni.h (working copy) @@ -15,11 +15,15 @@ #include "Settings.h" +#include "mpt/base/integer.hpp" +#include "mpt/string/types.hpp" #include "mpt/io_file_atomic/atomic_file.hpp" +#include #include #include #include +#include #include @@ -26,9 +30,49 @@ OPENMPT_NAMESPACE_BEGIN +struct IniVersion +{ + uint8 major = 0; + uint8 minor = 0; + uint8 patch = 0; +}; + + +enum class IniEncoding +{ + UTF32BE, + UTF32LE, + UTF16BE, + UTF16LE, + UTF8, + ANSI, +}; + + +enum class IniLineEnding +{ + CRLF, + LF, + CR, +}; + + +struct IniProbeResult +{ + IniVersion version; + IniEncoding encoding = IniEncoding::ANSI; + IniLineEnding lineending = IniLineEnding::CRLF; + bool case_sensitive = false; + bool has_bom = false; + mpt::IO::Offset data_offset = 0; +}; + + class IniFileHelpers { protected: + static IniProbeResult Probe(mpt::const_byte_span data); + static std::list> CreateIniHeader(IniVersion version, IniEncoding encoding); static mpt::winstring GetSection(const SettingPath &path); static mpt::winstring GetKey(const SettingPath &path); static mpt::ustring FormatValueAsIni(const SettingValue &value); @@ -36,19 +80,34 @@ }; -class WindowsIniFileBase +// Version 1: ANSI +// Version 2: UTF-16LE with BOM +// Version 3: UTF-8 with BOM (not supported by Windows) +// Version 4: UTF-16LE or UTF-8 with BOM and explicit version + + +class IniFileBase { protected: const mpt::PathString filename; mpt::IO::atomic_shared_file_ref file; protected: + IniFileBase(mpt::PathString filename_); + ~IniFileBase() = default; +public: + const mpt::PathString &Filename() const; + mpt::PathString GetFilename() const; +}; + + +class WindowsIniFileBase + : public IniFileBase +{ +protected: WindowsIniFileBase(mpt::PathString filename_); ~WindowsIniFileBase() = default; public: void ConvertToUnicode(const mpt::ustring &backupTag = mpt::ustring()); -public: - const mpt::PathString &Filename() const; - mpt::PathString GetFilename() const; }; @@ -105,6 +164,36 @@ }; +class CachedIniFileSettingsBackend + : public ISettingsBackend + , public IniFileBase + , protected IniFileHelpers +{ +private: + struct comments + { + std::vector before; + std::vector after; + }; +private: + std::map>>> cache; + std::map, comments> comments; +private: + +private: + void ReadFileIntoCache(); + void MergeSettingsIntoCache(const std::set &removeSections, const std::map> &settings); + void WriteCacheIntoFile(); +public: + CachedIniFileSettingsBackend(mpt::PathString filename_); + ~CachedIniFileSettingsBackend() override; +public: + virtual void InvalidateCache() override; + virtual SettingValue ReadSetting(const SettingPath &path, const SettingValue &def) const override; + virtual void WriteAllSettings(const std::set &removeSections, const std::map> &settings) override; +}; + + #ifndef IniFileSettingsBackend #define IniFileSettingsBackend ImmediateWindowsIniFileSettingsBackend #endif Index: test/test.cpp =================================================================== --- test/test.cpp (revision 24610) +++ test/test.cpp (working copy) @@ -3031,6 +3031,17 @@ #ifdef MODPLUG_TRACKER + { + const mpt::PathString in = P_("D:\\stuff\\in.ini"); + const mpt::PathString out = P_("D:\\stuff\\out.ini"); + CopyFile(in.AsNative().c_str(), out.AsNative().c_str(), FALSE); + { + FileSettingsContainer ini{out}; + ini.Write(U_("Dummy"), U_("Value1"), U_("\r\n")); + ini.Write(U_("Dummy"), U_("Value2"), U_(" \r\n ")); + } + } + VERIFY_EQUAL(SettingPath(U_("a"),U_("b")) < SettingPath(U_("a"),U_("c")), true); VERIFY_EQUAL(!(SettingPath(U_("c"),U_("b")) < SettingPath(U_("a"),U_("c"))), true); @@ -3042,6 +3053,8 @@ TestIniSettingsBackendRead(filename); + TestIniSettingsBackendRead(filename); + { IniFileSettingsContainer conf{filename}; int32 foobar = conf.Read(U_("Test"), U_("bar"), 23); @@ -3134,7 +3147,7 @@ { { - IniFileSettingsContainer conf{filename}; + FileSettingsContainer conf{filename}; conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("spacespacecharspacespace")), spacespacecharspacespace); conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("threespaces")), threespaces); conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("tab")), tab); @@ -3143,7 +3156,7 @@ conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("cc0")), cc0); } { - IniFileSettingsContainer conf{filename}; + FileSettingsContainer conf{filename}; //VERIFY_EQUAL(conf.Read(SettingPath(MPT_USTRING("Test"), MPT_USTRING("spacespacecharspacespace"))), spacespacecharspacespace); //VERIFY_EQUAL(conf.Read(SettingPath(MPT_USTRING("Test"), MPT_USTRING("threespaces"))), threespaces); //VERIFY_EQUAL(conf.Read(SettingPath(MPT_USTRING("Test"), MPT_USTRING("tab"))), tab); @@ -3160,6 +3173,28 @@ DeleteFile(mpt::support_long_path(filename.AsNative()).c_str()); } + { + { + FileSettingsContainer conf{filename}; + conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("spacespacecharspacespace")), spacespacecharspacespace); + conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("threespaces")), threespaces); + conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("tab")), tab); + conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("tokens")), tokens); + conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("xcrlfy")), xcrlfy); + conf.Write(SettingPath(MPT_USTRING("Test"), MPT_USTRING("cc0")), cc0); + } + { + FileSettingsContainer conf{filename}; + VERIFY_EQUAL(conf.Read(SettingPath(MPT_USTRING("Test"), MPT_USTRING("spacespacecharspacespace"))), spacespacecharspacespace); + VERIFY_EQUAL(conf.Read(SettingPath(MPT_USTRING("Test"), MPT_USTRING("threespaces"))), threespaces); + VERIFY_EQUAL(conf.Read(SettingPath(MPT_USTRING("Test"), MPT_USTRING("tab"))), tab); + VERIFY_EQUAL(conf.Read(SettingPath(MPT_USTRING("Test"), MPT_USTRING("tokens"))), tokens); + VERIFY_EQUAL(conf.Read(SettingPath(MPT_USTRING("Test"), MPT_USTRING("xcrlfy"))), xcrlfy); + VERIFY_EQUAL(conf.Read(SettingPath(MPT_USTRING("Test"), MPT_USTRING("cc0"))), cc0); + } + DeleteFile(mpt::support_long_path(filename.AsNative()).c_str()); + } + #endif // MODPLUG_TRACKER }