Fix crashing when hashing SHA256 for large files

Handle possible getFileContent exceptions and add currently missing possibility to signalize the file loading error.
(as returning an empty string may not be enough - e.g. for hash calculations, an empty file for hashing is also a valid case...)

Fix #17243, close #17252

PowerEditor/src/MISC/Common/Common.cpp

@@ -50,26 +50,53 @@ wstring commafyInt(size_t n)
 	return ss.str();
 }
 
-std::string getFileContent(const wchar_t *file2read)
+std::string getFileContent(const wchar_t* file2read, bool* pbFailed)
 {
+	if (pbFailed)
+		*pbFailed = false; // reset
+
 	if (!doesFileExist(file2read))
-		return "";
-
-	const size_t blockSize = 1024;
-	char data[blockSize];
-	std::string wholeFileContent = "";
-	FILE *fp = _wfopen(file2read, L"rb");
-	if (!fp)
-		return "";
-
-	size_t lenFile = 0;
-	do
 	{
-		lenFile = fread(data, 1, blockSize, fp);
+		if (pbFailed)
-		if (lenFile == 0) break;
+			*pbFailed = true;
-		wholeFileContent.append(data, lenFile);
+		return "";
+	}
+
+	FILE* fp = _wfopen(file2read, L"rb");
+	if (!fp)
+	{
+		if (pbFailed)
+			*pbFailed = true;
+		return "";
+	}
+	
+	static constexpr size_t blockSize = 1024 * 4; // 4K is optimal chunk for memory, cache, disk or network
+	char data[blockSize];
+	std::string wholeFileContent;
+	size_t lenFile = 0;
+	try
+	{
+		do
+		{
+			lenFile = fread(data, 1, blockSize, fp);
+			if (lenFile == 0) break;
+			wholeFileContent.append(data, lenFile);
+		} while (lenFile > 0);
+	}
+	catch ([[maybe_unused]] const std::bad_alloc& ex)
+	{
+		if (pbFailed)
+			*pbFailed = true;
+		std::string().swap(wholeFileContent); // to immediately release all the allocated memory
+		::MessageBoxW(NULL, L"std::bad_alloc exception caught!\n\nProbably not enough contiguous memory to complete the operation.",
+			L"Notepad++ - getFileContent", MB_OK | MB_ICONWARNING | MB_APPLMODAL);
+	}
+	catch (...)
+	{
+		if (pbFailed)
+			*pbFailed = true;
+		std::string().swap(wholeFileContent); // to immediately release all the allocated memory
 	}
-	while (lenFile > 0);
 
 	fclose(fp);
 	return wholeFileContent;

PowerEditor/src/MISC/Common/Common.h

@@ -56,7 +56,7 @@ std::string wstring2string(const std::wstring & rwString, UINT codepage);
 bool isInList(const wchar_t *token, const wchar_t *list);
 std::wstring BuildMenuFileName(int filenameLen, unsigned int pos, const std::wstring &filename, bool ordinalNumber = true);
 
-std::string getFileContent(const wchar_t *file2read);
+std::string getFileContent(const wchar_t* file2read, bool* pbFailed = nullptr);
 std::wstring relativeFilePathToFullFilePath(const wchar_t *relativeFilePath);
 void writeFileContent(const wchar_t *file2write, const char *content2write);
 bool matchInList(const wchar_t *fileName, const std::vector<std::wstring> & patterns);

PowerEditor/src/MISC/Common/verifySignedfile.cpp

@@ -72,7 +72,11 @@ bool SecurityGuard::checkSha256(const std::wstring& filePath, NppModule module2c
 		return true;
 	*/
 
-	std::string content = getFileContent(filePath.c_str());
+	bool bLoadingFailed = false;
+	std::string content = getFileContent(filePath.c_str(), &bLoadingFailed);
+	if (bLoadingFailed)
+		return false;
+
 	uint8_t sha2hash[32];
 	calc_sha_256(sha2hash, reinterpret_cast<const uint8_t*>(content.c_str()), content.length());
 

PowerEditor/src/MISC/md5/md5Dlgs.cpp

@@ -148,7 +148,10 @@ intptr_t CALLBACK HashFromFilesDlg::run_dlgProc(UINT message, WPARAM wParam, LPA
 							}
 							else
 							{
-								std::string content = getFileContent(it.c_str());
+								bool bLoadingFailed = false;
+								std::string content = getFileContent(it.c_str(), &bLoadingFailed);
+								if (bLoadingFailed)
+									return FALSE;
 
 								uint8_t hash[HASH_MAX_LENGTH]{};
 								wchar_t hashStr[HASH_STR_MAX_LENGTH]{};

PowerEditor/src/Notepad_plus_Window.cpp

@@ -376,24 +376,28 @@ void Notepad_plus_Window::init(HINSTANCE hInst, HWND parent, const wchar_t *cmdL
 		{
 			if (doesFileExist(cmdLineParams->_easterEggName.c_str()))
 			{
-				std::string content = getFileContent(cmdLineParams->_easterEggName.c_str());
+				bool bLoadingFailed = false;
-				WcharMbcsConvertor& wmc = WcharMbcsConvertor::getInstance();
+				std::string content = getFileContent(cmdLineParams->_easterEggName.c_str(), &bLoadingFailed);
-				_userQuote = wmc.char2wchar(content.c_str(), SC_CP_UTF8);
+				if (!bLoadingFailed)
-				if (!_userQuote.empty())
 				{
-					_quoteParams.reset();
+					WcharMbcsConvertor& wmc = WcharMbcsConvertor::getInstance();
-					_quoteParams._quote = _userQuote.c_str();
+					_userQuote = wmc.char2wchar(content.c_str(), SC_CP_UTF8);
-					_quoteParams._quoter = L"Anonymous #999";
+					if (!_userQuote.empty())
-					_quoteParams._shouldBeTrolling = false;
+					{
-					_quoteParams._lang = cmdLineParams->_langType;
+						_quoteParams.reset();
-					if (cmdLineParams->_ghostTypingSpeed == 1)
+						_quoteParams._quote = _userQuote.c_str();
-						_quoteParams._speed = QuoteParams::slow;
+						_quoteParams._quoter = L"Anonymous #999";
-					else if (cmdLineParams->_ghostTypingSpeed == 2)
+						_quoteParams._shouldBeTrolling = false;
-						_quoteParams._speed = QuoteParams::rapid;
+						_quoteParams._lang = cmdLineParams->_langType;
-					else if (cmdLineParams->_ghostTypingSpeed == 3)
+						if (cmdLineParams->_ghostTypingSpeed == 1)
-						_quoteParams._speed = QuoteParams::speedOfLight;
+							_quoteParams._speed = QuoteParams::slow;
+						else if (cmdLineParams->_ghostTypingSpeed == 2)
+							_quoteParams._speed = QuoteParams::rapid;
+						else if (cmdLineParams->_ghostTypingSpeed == 3)
+							_quoteParams._speed = QuoteParams::speedOfLight;
 
-					_notepad_plus_plus_core.showQuote(&_quoteParams);
+						_notepad_plus_plus_core.showQuote(&_quoteParams);
+					}
 				}
 			}
 		}

PowerEditor/src/Parameters.cpp

@@ -1238,10 +1238,14 @@ bool NppParameters::load()
 	if (_isCloud)
 	{
 		// Read cloud choice
-		std::string cloudChoiceStr = getFileContent(cloudChoicePath.c_str());
+		std::wstring cloudChoiceStrW = L"";
-		WcharMbcsConvertor& wmc = WcharMbcsConvertor::getInstance();
+		bool bLoadingFailed = false;
-		std::wstring cloudChoiceStrW = wmc.char2wchar(cloudChoiceStr.c_str(), SC_CP_UTF8);
+		std::string cloudChoiceStr = getFileContent(cloudChoicePath.c_str(), &bLoadingFailed);
-
+		if (!bLoadingFailed)
+		{
+			WcharMbcsConvertor& wmc = WcharMbcsConvertor::getInstance();
+			cloudChoiceStrW = wmc.char2wchar(cloudChoiceStr.c_str(), SC_CP_UTF8);
+		}
 		if (!cloudChoiceStrW.empty() && doesDirectoryExist(cloudChoiceStrW.c_str()))
 		{
 			_userPath = cloudChoiceStrW;