Update to scintilla 5.5.5 & Lexilla 5.4.3

Release 5.5.5 (https://www.scintilla.org/scintilla555.zip)

    Released 25 February 2025.
    Remember selection with undo and redo. Controlled with SCI_SETUNDOSELECTIONHISTORY. Feature #1273, Bug #1479, Bug #1224.
    Serialize selection type and ranges with SCI_GETSELECTIONSERIALIZED and SCI_SETSELECTIONSERIALIZED.
    For Win32, update Direct2D and DirectWrite interfaces used to 1.1 and add a lower-level approach to calling DirectWrite 1.1 by specifying SC_TECHNOLOGY_DIRECT_WRITE_1. Since Windows Vista does not support these API versions, Scintilla o longer supports DirectWrite on Windows Vista and will fall back to using GDI.
    Fix segmentation of long lexemes to avoid breaking before modifiers like accents that must be drawn with their base letters. For wrapping, try to break lines without separating letters from modifiers.
    For GTK on Windows, replace reverse arrow cursor with hand as reverse arrow was small in scaled modes. Bug #2460.
    Fix bug on Qt where double-click stopped working when Scintilla instance had been running for weeks.

Release 5.4.3 (https://www.scintilla.org/lexilla543.zip)

    Released 25 February 2025.
    C++: Fix evaluation of != in preprocessor condition. Issue #299.
    Modula-3: Allow digits in uppercase identifiers. Issue #297.
    Pascal: Fix asm style extending past end. Issue #295.
    Python: Fix detection of attributes and decorators. Issue #294, Pull request #302.
    Ruby: Implement substyles for identifiers SCE_RB_IDENTIFIER.
    Ruby: Recognize name as SCE_RB_DEFNAME in def when `::` used as well as `.`. Issue #300.

Close #16235

boostregex/BoostRegExSearch.cxx

@@ -1,12 +1,12 @@
 /**
  * Copyright (c) since 2009 Simon Steele - http://untidy.net/
  * Based on the work of Simon Steele for Programmer's Notepad 2 (http://untidy.net)
- * Converted from boost::xpressive to boost::regex and performance improvements 
+ * Converted from boost::xpressive to boost::regex and performance improvements
  * (principally caching the compiled regex), and support for UTF8 encoded text
  * (c) 2012 Dave Brotherstone - Changes for boost::regex
  * (c) 2013 Francois-R.Boyer@PolyMtl.ca - Empty match modes and best match backward search
  * (c) 2019 Don Ho - Adapt for upgrading Scitilla (to version 4.1.4) and boost (to version 1.70)
- * 
+ *
  */
 
 #include <stdlib.h>
@@ -15,6 +15,8 @@
 #include <string_view>
 #include <stdexcept>
 #include <optional>
+#include <map>
+#include <algorithm>
 
 #include "Scintilla.h"
 #include "ScintillaTypes.h"
@@ -54,16 +56,16 @@ class BoostRegexSearch : public RegexSearchBase
 {
 public:
 	BoostRegexSearch() {}
-	
+
 	virtual ~BoostRegexSearch()
 	{
 		delete[] _substituted;
 		_substituted = nullptr;
 	}
-	
+
 	virtual Sci::Position FindText(Document* doc, Sci::Position minPos, Sci::Position maxPos, const char *regex,
                         bool caseSensitive, bool word, bool wordStart, Scintilla::FindOption sciSearchFlags, Sci::Position *lengthRet) override;
-	
+
 	virtual const char *SubstituteByPosition(Document* doc, const char *text, Sci::Position *length) override;
 
 private:
@@ -82,38 +84,38 @@ private:
 			_position = -1;
 			return *this;
 		}
-		
+
 		void set(Document* document = NULL, Sci::Position position = -1, Sci::Position endPosition = -1) {
 			setDocument(document);
 			_position = position;
 			_endPositionForContinuationCheck = _endPosition = endPosition;
 			_documentModified = false;
 		}
-		
+
 		bool isContinuationSearch(Document* document, Sci::Position startPosition, int direction) {
 			if (hasDocumentChanged(document))
 				return false;
-			if (direction > 0) 
+			if (direction > 0)
 				return startPosition == _endPositionForContinuationCheck;
 			else
 				return startPosition == _position;
 		}
-		bool isEmpty() {
+		bool isEmpty() const {
 			return _position == _endPosition;
 		}
-		Sci::Position position() {
+		Sci::Position position() const {
 			return _position;
 		}
-		Sci::Position endPosition() {
+		Sci::Position endPosition() const {
 			return _endPosition;
 		}
-		Sci::Position length() {
+		Sci::Position length() const {
 			return _endPosition - _position;
 		}
-		int found() {
+		int found() const {
 			return _position >= 0;
 		}
-		
+
 	private:
 		bool hasDocumentChanged(Document* currentDocument) {
 			return currentDocument != _document || _documentModified;
@@ -128,7 +130,7 @@ private:
 					_document->AddWatcher(this, NULL);
 			}
 		}
-		
+
 		// DocWatcher, so we can track modifications to know if we should consider a search to be a continuation of last search:
 		virtual void NotifyModified(Document* modifiedDocument, DocModification mh, void* /*userData*/)
 		{
@@ -156,7 +158,7 @@ private:
 		{
 			if (deletedDocument == _document)
 			{
-				// We set the _document here, as we don't want to call the RemoveWatcher on this deleted document. 
+				// We set the _document here, as we don't want to call the RemoveWatcher on this deleted document.
 				// Calling RemoveWatcher inside NotifyDeleted results in a crash, as NotifyDeleted is called whilst
 				// iterating on the watchers list (since Scintilla 3.x).  Before 3.x, it was just a really bad idea.
 				_document = NULL;
@@ -168,13 +170,14 @@ private:
 		virtual void NotifyStyleNeeded(Document* /*document*/, void* /*userData*/, Sci::Position /*endPos*/) {}
 		virtual void NotifyLexerChanged(Document* /*document*/, void* /*userData*/) {}
 		virtual void NotifyErrorOccurred(Document* /*document*/, void* /*userData*/, Scintilla::Status /*status*/) {}
-		
+		virtual void NotifyGroupCompleted(Document* /*document*/, void* /*userData*/) noexcept {}
+
 		Document* _document;
 		bool _documentModified;
 		Sci::Position _position, _endPosition;
 		Sci::Position _endPositionForContinuationCheck;
 	};
-	
+
 	class CharTPtr { // Automatically translatable from utf8 to wchar_t*, if required, with allocation and deallocation on destruction; char* is not deallocated.
 	public:
 		CharTPtr(const char* ptr) : _charPtr(ptr), _wcharPtr(NULL) {}
@@ -209,20 +212,20 @@ private:
 		typedef CharT Char;
 		typedef basic_regex<CharT> Regex;
 		typedef match_results<CharacterIterator> MatchResults;
-		
+
 		MatchResults _match;
 	private:
 		Regex _regex;
 		std::string _lastRegexString;
 		int _lastCompileFlags;
 	};
-	
+
 	class SearchParameters {
 	public:
-		Sci::Position nextCharacter(Sci::Position position);
-		bool isLineStart(Sci::Position position);
-		bool isLineEnd(Sci::Position position);
-		
+		Sci::Position nextCharacter(Sci::Position position) const;
+		bool isLineStart(Sci::Position position) const;
+		bool isLineEnd(Sci::Position position) const;
+
 		Document* _document;
 		const char *_regexString;
 		int _compileFlags;
@@ -234,17 +237,17 @@ private:
 		bool _is_allowed_empty_at_start_position;
 		bool _skip_windows_line_end_as_one_character;
 	};
-	
+
 	static wchar_t *utf8ToWchar(const char *utf8);
 	static char    *wcharToUtf8(const wchar_t *w);
 	static char    *stringToCharPtr(const std::string& str);
 	static char    *stringToCharPtr(const std::wstring& str);
-	
+
 	EncodingDependent<char,    AnsiDocumentIterator> _ansi;
 	EncodingDependent<wchar_t, UTF8DocumentIterator> _utf8;
-	
+
 	char *_substituted = nullptr;
-	
+
 	Match _lastMatch;
 	int _lastDirection = 0;
 };
@@ -272,9 +275,9 @@ Sci::Position BoostRegexSearch::FindText(Document* doc, Sci::Position startPosit
 	g_exceptionMessage.clear();
 	try {
 		SearchParameters search{};
-		
+
 		search._document = doc;
-		
+
 		if (startPosition > endPosition
 			|| (startPosition == endPosition && _lastDirection < 0))  // If we search in an empty region, suppose the direction is the same as last search (this is only important to verify if there can be an empty match in that empty region).
 		{
@@ -293,26 +296,26 @@ Sci::Position BoostRegexSearch::FindText(Document* doc, Sci::Position startPosit
 		// Range endpoints should not be inside DBCS characters, but just in case, move them.
 		search._startPosition = doc->MovePositionOutsideChar(search._startPosition, 1, false);
 		search._endPosition = doc->MovePositionOutsideChar(search._endPosition, 1, false);
-		
+
 		const bool isUtf8 = (doc->CodePage() == SC_CP_UTF8);
-		search._compileFlags = 
+		search._compileFlags =
 			regex_constants::ECMAScript
 			| (caseSensitive ? 0 : regex_constants::icase);
 		search._regexString = regexString;
-		search._boostRegexFlags = 
+		search._boostRegexFlags =
 			((static_cast<int>(sciSearchFlags) & SCFIND_REGEXP_DOTMATCHESNL) ? regex_constants::match_default : regex_constants::match_not_dot_newline);
-		
+
 		const int empty_match_style = static_cast<int>(sciSearchFlags) & SCFIND_REGEXP_EMPTYMATCH_MASK;
 		const int allow_empty_at_start = static_cast<int>(sciSearchFlags) & SCFIND_REGEXP_EMPTYMATCH_ALLOWATSTART;
 
 		search._is_allowed_empty = empty_match_style != SCFIND_REGEXP_EMPTYMATCH_NONE;
-		search._is_allowed_empty_at_start_position = search._is_allowed_empty && 
+		search._is_allowed_empty_at_start_position = search._is_allowed_empty &&
 			(allow_empty_at_start
 			|| !_lastMatch.isContinuationSearch(doc, startPosition, search._direction)
 			|| (empty_match_style == SCFIND_REGEXP_EMPTYMATCH_ALL && !_lastMatch.isEmpty())	// If last match is empty and this is a continuation, then we would have same empty match at start position, if it was allowed.
 			);
 		search._skip_windows_line_end_as_one_character = (static_cast<int>(sciSearchFlags) & SCFIND_REGEXP_SKIPCRLFASONE) != 0;
-		
+
 		Match match =
 			isUtf8 ? _utf8.FindText(search)
 			       : _ansi.FindText(search);
@@ -394,7 +397,7 @@ BoostRegexSearch::Match BoostRegexSearch::EncodingDependent<CharT, CharacterIter
 	search._direction = 1;
 	const bool is_allowed_empty_at_end_position = search._is_allowed_empty_at_start_position;
 	search._is_allowed_empty_at_start_position = search._is_allowed_empty;
-	
+
 	MatchResults bestMatch;
 	Sci::Position bestPosition = -1;
 	Sci::Position bestEnd = -1;
@@ -429,7 +432,7 @@ void BoostRegexSearch::EncodingDependent<CharT, CharacterIterator>::compileRegex
 	}
 }
 
-Sci::Position BoostRegexSearch::SearchParameters::nextCharacter(Sci::Position position)
+Sci::Position BoostRegexSearch::SearchParameters::nextCharacter(Sci::Position position) const
 {
 	if (_skip_windows_line_end_as_one_character && _document->CharAt(position) == '\r' && _document->CharAt(position+1) == '\n')
 		return position + 2;
@@ -437,14 +440,14 @@ Sci::Position BoostRegexSearch::SearchParameters::nextCharacter(Sci::Position po
 		return _document->NextPosition(position, 1);
 }
 
-bool BoostRegexSearch::SearchParameters::isLineStart(Sci::Position position)
+bool BoostRegexSearch::SearchParameters::isLineStart(Sci::Position position) const
 {
 	return (position == 0)
 		|| _document->CharAt(position-1) == '\n'
 		|| (_document->CharAt(position-1) == '\r' && _document->CharAt(position) != '\n');
 }
 
-bool BoostRegexSearch::SearchParameters::isLineEnd(Sci::Position position)
+bool BoostRegexSearch::SearchParameters::isLineEnd(Sci::Position position) const
 {
 	return (position == _document->Length())
 		|| _document->CharAt(position) == '\r'

boostregex/UTF8DocumentIterator.cxx

@@ -18,6 +18,8 @@
 #include <string_view>
 #include <stdexcept>
 #include <optional>
+#include <map>
+#include <algorithm>
 
 #include "ILoader.h"
 #include "ILexer.h"

lexilla/.gitattributes

@@ -52,6 +52,7 @@
 **.nix text
 **.octave text
 **.p text
+**.pas text
 **.pl text
 **.p6 text
 **.ps1 text

lexilla/cppcheck.suppress

@@ -31,7 +31,6 @@ passedByValue
 missingIncludeSystem
 
 // Passing temporary string into hidden object creator functions: they do not hold the argument
-danglingTemporaryLifetime:lexilla/access/LexillaAccess.cxx
 returnDanglingLifetime:lexilla/access/LexillaAccess.cxx
 
 // cppcheck 2.11 limits checking of complex functions unless --check-level=exhaustive but that
@@ -45,6 +44,9 @@ checkLevelNormal:lexilla/lexers/LexRuby.cxx
 // Physically but not logically const.
 constVariablePointer:lexilla/examples/CheckLexilla/CheckLexilla.c
 
+// cppcheck appears to be incorrectly determining control flow: target_end is used after write
+unreadVariable:lexilla/lexers/LexRuby.cxx
+
 // Suppress most lexer warnings since the lexers are maintained by others
 redundantCondition:lexilla/lexers/LexA68k.cxx
 constParameterReference:lexilla/lexers/LexAbaqus.cxx
@@ -61,6 +63,7 @@ constParameterReference:lexilla/lexers/LexBash.cxx
 knownConditionTrueFalse:lexilla/lexers/LexBash.cxx
 variableScope:lexilla/lexers/LexBash.cxx
 constVariable:lexilla/lexers/LexBasic.cxx
+constParameterReference:lexilla/lexers/LexBullant.cxx
 constParameterReference:lexilla/lexers/LexCLW.cxx
 knownConditionTrueFalse:lexilla/lexers/LexCLW.cxx
 variableScope:lexilla/lexers/LexCmake.cxx
@@ -105,6 +108,7 @@ unreadVariable:lexilla/lexers/LexJulia.cxx
 variableScope:lexilla/lexers/LexJulia.cxx
 variableScope:lexilla/lexers/LexLaTeX.cxx
 constParameterReference:lexilla/lexers/LexLaTeX.cxx
+constParameterReference:lexilla/lexers/LexLisp.cxx
 constParameterPointer:lexilla/lexers/LexMagik.cxx
 constParameterReference:lexilla/lexers/LexMagik.cxx
 constParameterReference:lexilla/lexers/LexMarkdown.cxx
@@ -147,6 +151,8 @@ redundantInitialization:lexilla/lexers/LexRegistry.cxx
 constParameterReference:lexilla/lexers/LexRuby.cxx
 constParameterReference:lexilla/lexers/LexRust.cxx
 knownConditionTrueFalse:lexilla/lexers/LexScriptol.cxx
+constParameterReference:lexilla/lexers/LexScriptol.cxx
+constParameterPointer:lexilla/lexers/LexSmalltalk.cxx
 variableScope:lexilla/lexers/LexSpecman.cxx
 unreadVariable:lexilla/lexers/LexSpice.cxx
 constParameterReference:lexilla/lexers/LexSpice.cxx
@@ -157,6 +163,7 @@ clarifyCalculation:lexilla/lexers/LexTADS3.cxx
 constParameterReference:lexilla/lexers/LexTADS3.cxx
 constParameterReference:lexilla/lexers/LexTAL.cxx
 constVariableReference:lexilla/lexers/LexTCL.cxx
+constParameterPointer:lexilla/lexers/LexTCMD.cxx
 invalidscanf:lexilla/lexers/LexTCMD.cxx
 constParameterReference:lexilla/lexers/LexTeX.cxx
 variableScope:lexilla/lexers/LexTeX.cxx
@@ -166,6 +173,7 @@ constParameterReference:lexilla/lexers/LexVerilog.cxx
 variableScope:lexilla/lexers/LexVerilog.cxx
 badBitmaskCheck:lexilla/lexers/LexVerilog.cxx
 uselessCallsSubstr:lexilla/lexers/LexVerilog.cxx
+duplicateCondition:lexilla/lexers/LexVerilog.cxx
 constParameterReference:lexilla/lexers/LexVHDL.cxx
 constVariable:lexilla/lexers/LexVHDL.cxx
 shadowVariable:lexilla/lexers/LexVHDL.cxx
@@ -191,12 +199,60 @@ constParameterCallback:lexilla/lexers/LexPython.cxx
 constParameterCallback:lexilla/lexers/LexScriptol.cxx
 constParameterCallback:lexilla/lexers/LexVB.cxx
 
+constVariableReference:lexilla/lexers/LexA68k.cxx
+constVariableReference:lexilla/lexers/LexAPDL.cxx
+constVariableReference:lexilla/lexers/LexASY.cxx
+constVariableReference:lexilla/lexers/LexAVE.cxx
+constVariableReference:lexilla/lexers/LexAVS.cxx
+constVariableReference:lexilla/lexers/LexAU3.cxx
+constVariableReference:lexilla/lexers/LexAsn1.cxx
 constVariableReference:lexilla/lexers/LexBibTeX.cxx
+constVariableReference:lexilla/lexers/LexCaml.cxx
+constVariableReference:lexilla/lexers/LexCLW.cxx
+constVariableReference:lexilla/lexers/LexCmake.cxx
+constVariableReference:lexilla/lexers/LexCOBOL.cxx
+constVariableReference:lexilla/lexers/LexCoffeeScript.cxx
+constVariableReference:lexilla/lexers/LexCsound.cxx
 constVariableReference:lexilla/lexers/LexCSS.cxx
 constVariableReference:lexilla/lexers/LexCrontab.cxx
+constVariableReference:lexilla/lexers/LexDataflex.cxx
+constVariableReference:lexilla/lexers/LexDMAP.cxx
+constVariableReference:lexilla/lexers/LexECL.cxx
+constVariableReference:lexilla/lexers/LexEiffel.cxx
+constVariableReference:lexilla/lexers/LexEScript.cxx
+constVariableReference:lexilla/lexers/LexErlang.cxx
+constVariableReference:lexilla/lexers/LexFlagship.cxx
+constVariableReference:lexilla/lexers/LexForth.cxx
+constVariableReference:lexilla/lexers/LexFortran.cxx
+constVariableReference:lexilla/lexers/LexGAP.cxx
 constVariableReference:lexilla/lexers/LexGui4Cli.cxx
+constVariableReference:lexilla/lexers/LexKix.cxx
+constVariableReference:lexilla/lexers/LexKVIrc.cxx
+constVariableReference:lexilla/lexers/LexLout.cxx
+constVariableReference:lexilla/lexers/LexMagik.cxx
+constVariableReference:lexilla/lexers/LexMatlab.cxx
 constVariableReference:lexilla/lexers/LexMetapost.cxx
+constVariableReference:lexilla/lexers/LexMMIXAL.cxx
+constVariableReference:lexilla/lexers/LexMSSQL.cxx
+constVariableReference:lexilla/lexers/LexNsis.cxx
 constVariableReference:lexilla/lexers/LexOpal.cxx
+constVariableReference:lexilla/lexers/LexPOV.cxx
+constVariableReference:lexilla/lexers/LexPascal.cxx
+constVariableReference:lexilla/lexers/LexPB.cxx
+constVariableReference:lexilla/lexers/LexPowerPro.cxx
+constVariableReference:lexilla/lexers/LexPS.cxx
+constVariableReference:lexilla/lexers/LexRebol.cxx
+constVariableReference:lexilla/lexers/LexSAS.cxx
+constVariableReference:lexilla/lexers/LexSML.cxx
+constVariableReference:lexilla/lexers/LexSorcus.cxx
+constVariableReference:lexilla/lexers/LexSpecman.cxx
+constVariableReference:lexilla/lexers/LexStata.cxx
+constVariableReference:lexilla/lexers/LexTACL.cxx
+constVariableReference:lexilla/lexers/LexTADS3.cxx
+constVariableReference:lexilla/lexers/LexTAL.cxx
+constVariableReference:lexilla/lexers/LexTCMD.cxx
+constVariableReference:lexilla/lexers/LexTeX.cxx
+constVariableReference:lexilla/lexers/LexVHDL.cxx
 
 // Suppress everything in test example files
 *:lexilla/test/examples/*

lexilla/doc/Lexilla.html

@@ -9,7 +9,7 @@
     <meta name="keywords" content="Scintilla, SciTE, Editing Component, Text Editor" />
     <meta name="Description"
     content="www.scintilla.org is the home of the Scintilla editing component and SciTE text editor application." />
-    <meta name="Date.Modified" content="20241218" />
+    <meta name="Date.Modified" content="20250225" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <style type="text/css">
         .logo {
@@ -61,8 +61,8 @@
           <font color="#FFCC99" size="4"> A library of language lexers for use with Scintilla</font>
         </td>
         <td width="40%" align="right">
-          <font color="#FFCC99" size="3">Release version 5.4.2<br />
-           Site last modified December 18 2024</font>
+          <font color="#FFCC99" size="3">Release version 5.4.3<br />
+           Site last modified February 25 2025</font>
         </td>
         <td width="20%">
           &nbsp;
@@ -77,11 +77,11 @@
       </tr>
     </table>
     <ul id="versionlist">
+      <li>Version 5.4.3 improves C++, Modula 3, Pascal, Python, and Ruby.</li>
       <li>Version 5.4.2 adds Nix lexer. Improves JavaScript, PHP, Rust, TOML, and Zig.</li>
       <li>Version 5.4.1 adds Dart, troff, and Zig lexers. Improves C++, F#, HTML, and Smalltalk.</li>
       <li>Version 5.4.0 adds a TOML lexer.</li>
       <li>Version 5.3.3 improves HTML, JavaScript, Lua, PHP, and XML.</li>
-      <li>Version 5.3.2 improves COBOL, HTML, Lua, Ruby, and Rust.</li>
     </ul>
     <ul id="menu">
       <li id="remote1"><a href="https://www.scintilla.org/SciTEImage.html">Screenshot</a></li>

lexilla/doc/LexillaDownload.html

@@ -26,9 +26,9 @@
     <table bgcolor="#CCCCCC" width="100%" cellspacing="0" cellpadding="8" border="0">
       <tr>
         <td>
-          <font size="4"> <a href="https://www.scintilla.org/lexilla542.zip">
+          <font size="4"> <a href="https://www.scintilla.org/lexilla543.zip">
 	Windows</a>&nbsp;&nbsp;
-	<a href="https://www.scintilla.org/lexilla542.tgz">
+	<a href="https://www.scintilla.org/lexilla543.tgz">
           GTK/Linux</a>&nbsp;&nbsp;
 	</font>
         </td>
@@ -42,7 +42,7 @@
       containing very few restrictions.
     </p>
     <h3>
-       Release 5.4.2
+       Release 5.4.3
     </h3>
     <h4>
        Source Code
@@ -50,8 +50,8 @@
        The source code package contains all of the source code for Lexilla but no binary
 	executable code and is available in
        <ul>
-       <li><a href="https://www.scintilla.org/lexilla542.zip">zip format</a> (1.4M) commonly used on Windows</li>
-       <li><a href="https://www.scintilla.org/lexilla542.tgz">tgz format</a> (1.0M) commonly used on Linux and compatible operating systems</li>
+       <li><a href="https://www.scintilla.org/lexilla543.zip">zip format</a> (1.4M) commonly used on Windows</li>
+       <li><a href="https://www.scintilla.org/lexilla543.tgz">tgz format</a> (1.0M) commonly used on Linux and compatible operating systems</li>
        </ul>
        Instructions for building on both Windows and Linux are included in the readme file.
     <h4>

lexilla/doc/LexillaHistory.html

@@ -37,6 +37,7 @@
        Lexilla was originally code that was part of the Scintilla project.
        Thus it shares much of the history and contributors of Scintilla before it was extracted as its own project.
     </p>
+    <a href="#Releases">Releases</a>
     <h2>Contributors</h2>
     <p>
        Thanks to all the people that have contributed patches, bug reports and suggestions.
@@ -588,9 +589,42 @@
       </tr><tr>
 	<td>Henrik S. Johansen</td>
 	<td>Ekopalypse</td>
+	<td>HoTschir</td>
     </tr>
     </table>
-    <h2>Releases</h2>
+    <h2 id="Releases">Releases</h2>
+    <h3>
+       <a href="https://www.scintilla.org/lexilla543.zip">Release 5.4.3</a>
+    </h3>
+    <ul>
+	<li>
+	Released 25 February 2025.
+	</li>
+	<li>
+	C++: Fix evaluation of != in preprocessor condition.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/299">Issue #299</a>.
+	</li>
+	<li>
+	Modula-3: Allow digits in uppercase identifiers.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/297">Issue #297</a>.
+	</li>
+	<li>
+	Pascal: Fix asm style extending past end.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/295">Issue #295</a>.
+	</li>
+	<li>
+	Python: Fix detection of attributes and decorators.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/294">Issue #294</a>,
+	<a href="https://github.com/ScintillaOrg/lexilla/pull/302">Pull request #302</a>.
+	</li>
+	<li>
+	Ruby: Implement substyles for identifiers SCE_RB_IDENTIFIER.
+	</li>
+	<li>
+	Ruby: Recognize name as SCE_RB_DEFNAME in def when `::` used as well as `.`.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/300">Issue #300</a>.
+	</li>
+    </ul>
     <h3>
        <a href="https://www.scintilla.org/lexilla542.zip">Release 5.4.2</a>
     </h3>

lexilla/lexers/LexCPP.cxx

@@ -496,7 +496,7 @@ struct OptionSetCPP : public OptionSet<OptionsCPP> {
 
 const char styleSubable[] = {SCE_C_IDENTIFIER, SCE_C_COMMENTDOCKEYWORD, 0};
 
-LexicalClass lexicalClasses[] = {
+const LexicalClass lexicalClasses[] = {
 	// Lexer Cpp SCLEX_CPP SCE_C_:
 	0, "SCE_C_DEFAULT", "default", "White space",
 	1, "SCE_C_COMMENT", "comment", "Comment: /* */.",
@@ -528,7 +528,7 @@ LexicalClass lexicalClasses[] = {
 	27, "SCE_C_ESCAPESEQUENCE", "literal string escapesequence", "Escape sequence",
 };
 
-const int sizeLexicalClasses = static_cast<int>(std::size(lexicalClasses));
+constexpr int sizeLexicalClasses{ std::size(lexicalClasses) };
 
 }
 
@@ -1725,7 +1725,7 @@ void LexerCPP::EvaluateTokens(Tokens &tokens, const SymbolTable &preprocessorDef
 
 	// Evaluate logical negations
 	for (size_t j=0; (j+1)<tokens.size();) {
-		if (setNegationOp.Contains(tokens[j][0])) {
+		if (setNegationOp.Contains(tokens[j][0]) && (tokens[j] != "!=")) {
 			int isTrue = atoi(tokens[j+1].c_str());
 			if (tokens[j] == "!")
 				isTrue = !isTrue;

lexilla/lexers/LexModula.cxx

@@ -366,7 +366,7 @@ static void ColouriseModulaDoc(	Sci_PositionU startPos,
 				if( isupper( sc.ch ) && isupper( sc.chNext ) ) {
 					for( i = 0; i < BUFLEN - 1; i++ ) {
 						buf[i] = sc.GetRelative(i);
-						if( !isalpha( buf[i] ) && !(buf[i] == '_') )
+						if( !(IsAlphaNumeric( buf[i] ) || buf[i] == '_') )
 							break;
 					}
 					kl = i;

lexilla/lexers/LexPascal.cxx

@@ -133,7 +133,9 @@ contains requires
 
 using namespace Lexilla;
 
-static void GetRangeLowered(Sci_PositionU start,
+namespace {
+
+void GetRangeLowered(Sci_PositionU start,
 		Sci_PositionU end,
 		Accessor &styler,
 		char *s,
@@ -146,7 +148,7 @@ static void GetRangeLowered(Sci_PositionU start,
 	s[i] = '\0';
 }
 
-static void GetForwardRangeLowered(Sci_PositionU start,
+void GetForwardRangeLowered(Sci_PositionU start,
 		CharacterSet &charSet,
 		Accessor &styler,
 		char *s,
@@ -170,7 +172,7 @@ enum {
 	stateFoldMaskAll = 0x0FFF
 };
 
-static void ClassifyPascalWord(WordList *keywordlists[], StyleContext &sc, int &curLineState, bool bSmartHighlighting) {
+void ClassifyPascalWord(WordList *keywordlists[], StyleContext &sc, int &curLineState, bool bSmartHighlighting) {
 	WordList& keywords = *keywordlists[0];
 
 	char s[100];
@@ -215,7 +217,7 @@ static void ClassifyPascalWord(WordList *keywordlists[], StyleContext &sc, int &
 	sc.SetState(SCE_PAS_DEFAULT);
 }
 
-static void ColourisePascalDoc(Sci_PositionU startPos, Sci_Position length, int initStyle, WordList *keywordlists[],
+void ColourisePascalDoc(Sci_PositionU startPos, Sci_Position length, int initStyle, WordList *keywordlists[],
 		Accessor &styler) {
 	bool bSmartHighlighting = styler.GetPropertyInt("lexer.pascal.smart.highlighting", 1) != 0;
 
@@ -225,18 +227,14 @@ static void ColourisePascalDoc(Sci_PositionU startPos, Sci_Position length, int
 	CharacterSet setHexNumber(CharacterSet::setDigits, "abcdefABCDEF");
 	CharacterSet setOperator(CharacterSet::setNone, "#$&'()*+,-./:;<=>@[]^{}");
 
-	Sci_Position curLine = styler.GetLine(startPos);
-	int curLineState = curLine > 0 ? styler.GetLineState(curLine - 1) : 0;
+	int curLineState = 0;
 
 	StyleContext sc(startPos, length, initStyle, styler);
+	if (sc.currentLine > 0) {
+		curLineState = styler.GetLineState(sc.currentLine - 1);
+	}
 
 	for (; sc.More(); sc.Forward()) {
-		if (sc.atLineEnd) {
-			// Update the line state, so it can be seen by next line
-			curLine = styler.GetLine(sc.currentPos);
-			styler.SetLineState(curLine, curLineState);
-		}
-
 		// Determine if the current state should terminate.
 		switch (sc.state) {
 			case SCE_PAS_NUMBER:
@@ -335,6 +333,11 @@ static void ColourisePascalDoc(Sci_PositionU startPos, Sci_Position length, int
 				sc.SetState(SCE_PAS_ASM);
 			}
 		}
+
+		if (sc.atLineEnd) {
+			// Update the line state, so it can be seen by next line
+			styler.SetLineState(sc.currentLine, curLineState);
+		}
 	}
 
 	if (sc.state == SCE_PAS_IDENTIFIER && setWord.Contains(sc.chPrev)) {
@@ -344,11 +347,11 @@ static void ColourisePascalDoc(Sci_PositionU startPos, Sci_Position length, int
 	sc.Complete();
 }
 
-static bool IsStreamCommentStyle(int style) {
+bool IsStreamCommentStyle(int style) {
 	return style == SCE_PAS_COMMENT || style == SCE_PAS_COMMENT2;
 }
 
-static bool IsCommentLine(Sci_Position line, Accessor &styler) {
+bool IsCommentLine(Sci_Position line, Accessor &styler) {
 	Sci_Position pos = styler.LineStart(line);
 	Sci_Position eolPos = styler.LineStart(line + 1) - 1;
 	for (Sci_Position i = pos; i < eolPos; i++) {
@@ -364,16 +367,16 @@ static bool IsCommentLine(Sci_Position line, Accessor &styler) {
 	return false;
 }
 
-static unsigned int GetFoldInPreprocessorLevelFlag(int lineFoldStateCurrent) {
+unsigned int GetFoldInPreprocessorLevelFlag(int lineFoldStateCurrent) {
 	return lineFoldStateCurrent & stateFoldInPreprocessorLevelMask;
 }
 
-static void SetFoldInPreprocessorLevelFlag(int &lineFoldStateCurrent, unsigned int nestLevel) {
+void SetFoldInPreprocessorLevelFlag(int &lineFoldStateCurrent, unsigned int nestLevel) {
 	lineFoldStateCurrent &= ~stateFoldInPreprocessorLevelMask;
 	lineFoldStateCurrent |= nestLevel & stateFoldInPreprocessorLevelMask;
 }
 
-static void ClassifyPascalPreprocessorFoldPoint(int &levelCurrent, int &lineFoldStateCurrent,
+void ClassifyPascalPreprocessorFoldPoint(int &levelCurrent, int &lineFoldStateCurrent,
 		Sci_PositionU startPos, Accessor &styler) {
 	CharacterSet setWord(CharacterSet::setAlpha);
 
@@ -406,7 +409,7 @@ static void ClassifyPascalPreprocessorFoldPoint(int &levelCurrent, int &lineFold
 	}
 }
 
-static Sci_PositionU SkipWhiteSpace(Sci_PositionU currentPos, Sci_PositionU endPos,
+Sci_PositionU SkipWhiteSpace(Sci_PositionU currentPos, Sci_PositionU endPos,
 		Accessor &styler, bool includeChars = false) {
 	CharacterSet setWord(CharacterSet::setAlphaNum, "_");
 	Sci_PositionU j = currentPos + 1;
@@ -419,7 +422,7 @@ static Sci_PositionU SkipWhiteSpace(Sci_PositionU currentPos, Sci_PositionU endP
 	return j;
 }
 
-static void ClassifyPascalWordFoldPoint(int &levelCurrent, int &lineFoldStateCurrent,
+void ClassifyPascalWordFoldPoint(int &levelCurrent, int &lineFoldStateCurrent,
 		Sci_Position startPos, Sci_PositionU endPos,
 		Sci_PositionU lastStart, Sci_PositionU currentPos, Accessor &styler) {
 	char s[100];
@@ -517,7 +520,7 @@ static void ClassifyPascalWordFoldPoint(int &levelCurrent, int &lineFoldStateCur
 	}
 }
 
-static void FoldPascalDoc(Sci_PositionU startPos, Sci_Position length, int initStyle, WordList *[],
+void FoldPascalDoc(Sci_PositionU startPos, Sci_Position length, int initStyle, WordList *[],
 		Accessor &styler) {
 	bool foldComment = styler.GetPropertyInt("fold.comment") != 0;
 	bool foldPreprocessor = styler.GetPropertyInt("fold.preprocessor") != 0;
@@ -608,9 +611,11 @@ static void FoldPascalDoc(Sci_PositionU startPos, Sci_Position length, int initS
 	styler.SetLevel(lineCurrent, lev);
 }
 
-static const char * const pascalWordListDesc[] = {
+const char * const pascalWordListDesc[] = {
 	"Keywords",
 	0
 };
 
+}
+
 extern const LexerModule lmPascal(SCLEX_PASCAL, ColourisePascalDoc, "pascal", FoldPascalDoc, pascalWordListDesc);

lexilla/lexers/LexPython.cxx

@@ -714,9 +714,9 @@ void SCI_METHOD LexerPython::Lex(Sci_PositionU startPos, Sci_Position length, in
 							pos--;
 							ch = styler.SafeGetCharAt(pos, '\0');
 						}
-						if (pos < 0 || ch == '.') {
+						if (ch == '.') {
 							// Is this an attribute we could style? if it is, do as asked
-							bool isComment = false;
+							bool isComment = AnyOf(styler.BufferStyleAt(pos), SCE_P_COMMENTLINE, SCE_P_COMMENTBLOCK);
 							bool isDecoratorAttribute = false;
 							const Sci_Position attrLine = styler.GetLine(pos);
 							for (Sci_Position i = styler.LineStart(attrLine); i < pos; i++) {
@@ -726,7 +726,7 @@ void SCI_METHOD LexerPython::Lex(Sci_PositionU startPos, Sci_Position length, in
 								if (attrCh == '#')
 									isComment = true;
 								// Detect if this attribute belongs to a decorator
-								if (!IsASpaceOrTab(ch))
+								if (!IsASpaceOrTab(attrCh))
 									break;
 							}
 							if (((isDecoratorAttribute) && (!isComment)) && (((options.decoratorAttributes == 1)  && (style == SCE_P_IDENTIFIER)) || (options.decoratorAttributes == 2))) {

lexilla/lexlib/StyleContext.cxx

@@ -54,6 +54,8 @@ bool StyleContext::MatchIgnoreCase(const char *s) {
 	if (MakeLowerCase(ch) != static_cast<unsigned char>(*s))
 		return false;
 	s++;
+	if (!*s)
+		return true;
 	if (MakeLowerCase(chNext) != static_cast<unsigned char>(*s))
 		return false;
 	s++;

lexilla/src/Lexilla/Info.plist

@@ -15,7 +15,7 @@
 	<key>CFBundlePackageType</key>
 	<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
 	<key>CFBundleShortVersionString</key>
-	<string>5.4.2</string>
+	<string>5.4.3</string>
 	<key>CFBundleVersion</key>
 	<string>$(CURRENT_PROJECT_VERSION)</string>
 	<key>NSHumanReadableCopyright</key>

lexilla/src/Lexilla/Lexilla.xcodeproj/project.pbxproj

@@ -876,7 +876,7 @@
 			buildSettings = {
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 5.4.2;
+				CURRENT_PROJECT_VERSION = 5.4.3;
 				DEAD_CODE_STRIPPING = YES;
 				DEVELOPMENT_TEAM = 4F446KW87E;
 				DYLIB_COMPATIBILITY_VERSION = 1;
@@ -904,7 +904,7 @@
 			buildSettings = {
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 5.4.2;
+				CURRENT_PROJECT_VERSION = 5.4.3;
 				DEAD_CODE_STRIPPING = YES;
 				DEVELOPMENT_TEAM = 4F446KW87E;
 				DYLIB_COMPATIBILITY_VERSION = 1;

lexilla/src/LexillaVersion.rc

@@ -4,8 +4,8 @@
 
 #include <windows.h>
 
-#define VERSION_LEXILLA "5.4.2"
-#define VERSION_WORDS 5, 4, 2, 0
+#define VERSION_LEXILLA "5.4.3"
+#define VERSION_WORDS 5, 4, 3, 0
 
 VS_VERSION_INFO VERSIONINFO
 FILEVERSION	VERSION_WORDS

lexilla/src/deps.mak

@@ -1304,12 +1304,16 @@ $(DIR_O)/LexRuby.o: \
 	../../scintilla/include/Sci_Position.h \
 	../../scintilla/include/Scintilla.h \
 	../include/SciLexer.h \
+	../lexlib/InList.h \
 	../lexlib/WordList.h \
 	../lexlib/LexAccessor.h \
 	../lexlib/Accessor.h \
 	../lexlib/StyleContext.h \
 	../lexlib/CharacterSet.h \
-	../lexlib/LexerModule.h
+	../lexlib/LexerModule.h \
+	../lexlib/OptionSet.h \
+	../lexlib/SubStyles.h \
+	../lexlib/DefaultLexer.h
 $(DIR_O)/LexRust.o: \
 	../lexers/LexRust.cxx \
 	../../scintilla/include/ILexer.h \

lexilla/src/nmdeps.mak

@@ -1304,12 +1304,16 @@ $(DIR_O)/LexRuby.obj: \
 	../../scintilla/include/Sci_Position.h \
 	../../scintilla/include/Scintilla.h \
 	../include/SciLexer.h \
+	../lexlib/InList.h \
 	../lexlib/WordList.h \
 	../lexlib/LexAccessor.h \
 	../lexlib/Accessor.h \
 	../lexlib/StyleContext.h \
 	../lexlib/CharacterSet.h \
-	../lexlib/LexerModule.h
+	../lexlib/LexerModule.h \
+	../lexlib/OptionSet.h \
+	../lexlib/SubStyles.h \
+	../lexlib/DefaultLexer.h
 $(DIR_O)/LexRust.obj: \
 	../lexers/LexRust.cxx \
 	../../scintilla/include/ILexer.h \

lexilla/test/examples/cpp/x.cxx

@@ -66,6 +66,11 @@ b
 // Active
 #endif
 
+#define ONE 1
+#if ONE != 1
+// Inactive
+#endif
+
 int main() {
 	double x[] = {3.14159,6.02e23,1.6e-19,1.0+1};
 	int y[] = {75,0113,0x4b};

lexilla/test/examples/cpp/x.cxx.folded

@@ -66,6 +66,11 @@
  0 401 401 | // Active
  0 401 400 | #endif
  1 400 400   
+ 0 400 400   #define ONE 1
+ 2 400 401 + #if ONE != 1
+ 0 401 401 | // Inactive
+ 0 401 400 | #endif
+ 1 400 400   
  2 400 401 + int main() {
  0 401 401 | 	double x[] = {3.14159,6.02e23,1.6e-19,1.0+1};
  0 401 401 | 	int y[] = {75,0113,0x4b};

lexilla/test/examples/cpp/x.cxx.styled

@@ -66,6 +66,11 @@
 {2}// Active
 {9}#endif
 {0}
+{9}#define ONE 1
+#if ONE != 1
+{66}// Inactive
+{9}#endif
+{0}
 {5}int{0} {11}main{10}(){0} {10}{{0}
 	{11}double{0} {11}x{10}[]{0} {10}={0} {10}{{4}3.14159{10},{4}6.02e23{10},{4}1.6e-19{10},{4}1.0{10}+{4}1{10}};{0}
 	{5}int{0} {11}y{10}[]{0} {10}={0} {10}{{4}75{10},{4}0113{10},{4}0x4b{10}};{0}

lexilla/test/examples/modula/Issue297.m3

@@ -0,0 +1,6 @@
+INTERFACE Issue297;
+
+TYPE
+  INTEGER32 = [-16_7fffffff-1 .. 16_7fffffff];
+
+END Issue297.

lexilla/test/examples/modula/Issue297.m3.folded

@@ -0,0 +1,7 @@
+ 0 400 400   INTERFACE Issue297;
+ 1 400 400   
+ 0 400 400   TYPE
+ 0 400 400     INTEGER32 = [-16_7fffffff-1 .. 16_7fffffff];
+ 1 400 400   
+ 0 400 3ff   END Issue297.
+ 1 3ff 3ff   

lexilla/test/examples/modula/Issue297.m3.styled

@@ -0,0 +1,6 @@
+{4}INTERFACE{0} Issue297{16};{0}
+
+{4}TYPE{0}
+  INTEGER32 {16}={0} {16}[-{7}16_7fffffff{16}-{6}1{0} {16}..{0} {7}16_7fffffff{16}];{0}
+
+{4}END{0} Issue297{16}.{0}

lexilla/test/examples/pascal/AllStyles.pas

@@ -0,0 +1,82 @@
+// Enumerate all primary styles: 0 to 14
+
+{
+  SCE_PAS_DEFAULT=0
+  SCE_PAS_IDENTIFIER=1
+  SCE_PAS_COMMENT=2
+  SCE_PAS_COMMENT2=3
+  SCE_PAS_COMMENTLINE=4
+  SCE_PAS_PREPROCESSOR=5
+  SCE_PAS_PREPROCESSOR2=6
+  SCE_PAS_NUMBER=7
+  SCE_PAS_HEXNUMBER=8
+  SCE_PAS_WORD=9
+  SCE_PAS_STRING=10
+  SCE_PAS_STRINGEOL=11
+  SCE_PAS_CHARACTER=12
+  SCE_PAS_OPERATOR=13
+  SCE_PAS_ASM=14
+}
+
+// default=0
+   
+// identifier=1
+function functionname(var paramerter1: type1):result1;
+procedure procedurename(const parameter2: type2);
+
+// comment=2
+{comment text}
+
+// comment2=3
+(* comment text *)
+
+// commentline=4 
+// example line
+
+// preprocessor=5
+{$DEFINE xyz}
+
+{$IFDEF xyz}
+   codeblock 1
+{$else}
+   codeblock 2
+{$endif}
+
+// preprocessor2=6
+(*$DEFINE xyz*)
+
+// number=7
+123
+1.23
+-123
+-12.3
++123
+123
+1.23e2
+-1.23E2
+
+// hexnumber=8
+$123
+$123ABCDEF
+$ABCDEF123
+
+// word=9
+absolute abstract and array as 
+
+// string=10
+'string'
+
+// stringeol=11
+'string
+
+// character=12
+#65
+
+// operator=13 
+$ & * + / < = > ^  
+
+// asm
+asm 
+  this is 
+  inside assembler
+end

lexilla/test/examples/pascal/AllStyles.pas.folded

@@ -0,0 +1,83 @@
+ 0 400   0   // Enumerate all primary styles: 0 to 14
+ 1 400   0   
+ 2 400   0 + {
+ 0 401   0 |   SCE_PAS_DEFAULT=0
+ 0 401   0 |   SCE_PAS_IDENTIFIER=1
+ 0 401   0 |   SCE_PAS_COMMENT=2
+ 0 401   0 |   SCE_PAS_COMMENT2=3
+ 0 401   0 |   SCE_PAS_COMMENTLINE=4
+ 0 401   0 |   SCE_PAS_PREPROCESSOR=5
+ 0 401   0 |   SCE_PAS_PREPROCESSOR2=6
+ 0 401   0 |   SCE_PAS_NUMBER=7
+ 0 401   0 |   SCE_PAS_HEXNUMBER=8
+ 0 401   0 |   SCE_PAS_WORD=9
+ 0 401   0 |   SCE_PAS_STRING=10
+ 0 401   0 |   SCE_PAS_STRINGEOL=11
+ 0 401   0 |   SCE_PAS_CHARACTER=12
+ 0 401   0 |   SCE_PAS_OPERATOR=13
+ 0 401   0 |   SCE_PAS_ASM=14
+ 0 401   0 | }
+ 1 400   0   
+ 0 400   0   // default=0
+ 1 400   0      
+ 0 400   0   // identifier=1
+ 0 400   0   function functionname(var paramerter1: type1):result1;
+ 0 400   0   procedure procedurename(const parameter2: type2);
+ 1 400   0   
+ 0 400   0   // comment=2
+ 0 400   0   {comment text}
+ 1 400   0   
+ 0 400   0   // comment2=3
+ 0 400   0   (* comment text *)
+ 1 400   0   
+ 2 400   0 + // commentline=4 
+ 0 401   0 | // example line
+ 1 400   0   
+ 0 400   0   // preprocessor=5
+ 0 400   0   {$DEFINE xyz}
+ 1 400   0   
+ 2 400   0 + {$IFDEF xyz}
+ 0 401   0 |    codeblock 1
+ 0 401   0 | {$else}
+ 0 401   0 |    codeblock 2
+ 0 401   0 | {$endif}
+ 1 400   0   
+ 0 400   0   // preprocessor2=6
+ 0 400   0   (*$DEFINE xyz*)
+ 1 400   0   
+ 0 400   0   // number=7
+ 0 400   0   123
+ 0 400   0   1.23
+ 0 400   0   -123
+ 0 400   0   -12.3
+ 0 400   0   +123
+ 0 400   0   123
+ 0 400   0   1.23e2
+ 0 400   0   -1.23E2
+ 1 400   0   
+ 0 400   0   // hexnumber=8
+ 0 400   0   $123
+ 0 400   0   $123ABCDEF
+ 0 400   0   $ABCDEF123
+ 1 400   0   
+ 0 400   0   // word=9
+ 0 400   0   absolute abstract and array as 
+ 1 400   0   
+ 0 400   0   // string=10
+ 0 400   0   'string'
+ 1 400   0   
+ 0 400   0   // stringeol=11
+ 0 400   0   'string
+ 1 400   0   
+ 0 400   0   // character=12
+ 0 400   0   #65
+ 1 400   0   
+ 0 400   0   // operator=13 
+ 0 400   0   $ & * + / < = > ^  
+ 1 400   0   
+ 0 400   0   // asm
+ 2 400   0 + asm 
+ 0 401   0 |   this is 
+ 0 401   0 |   inside assembler
+ 0 401   0 | end
+ 1 400   0   

lexilla/test/examples/pascal/AllStyles.pas.styled

@@ -0,0 +1,82 @@
+{4}// Enumerate all primary styles: 0 to 14
+{0}
+{2}{
+  SCE_PAS_DEFAULT=0
+  SCE_PAS_IDENTIFIER=1
+  SCE_PAS_COMMENT=2
+  SCE_PAS_COMMENT2=3
+  SCE_PAS_COMMENTLINE=4
+  SCE_PAS_PREPROCESSOR=5
+  SCE_PAS_PREPROCESSOR2=6
+  SCE_PAS_NUMBER=7
+  SCE_PAS_HEXNUMBER=8
+  SCE_PAS_WORD=9
+  SCE_PAS_STRING=10
+  SCE_PAS_STRINGEOL=11
+  SCE_PAS_CHARACTER=12
+  SCE_PAS_OPERATOR=13
+  SCE_PAS_ASM=14
+}{0}
+
+{4}// default=0
+{0}   
+{4}// identifier=1
+{9}function{0} {1}functionname{13}({9}var{0} {1}paramerter1{13}:{0} {1}type1{13}):{1}result1{13};{0}
+{9}procedure{0} {1}procedurename{13}({9}const{0} {1}parameter2{13}:{0} {1}type2{13});{0}
+
+{4}// comment=2
+{2}{comment text}{0}
+
+{4}// comment2=3
+{3}(* comment text *){0}
+
+{4}// commentline=4 
+// example line
+{0}
+{4}// preprocessor=5
+{5}{$DEFINE xyz}{0}
+
+{5}{$IFDEF xyz}{0}
+   {1}codeblock{0} {7}1{0}
+{5}{$else}{0}
+   {1}codeblock{0} {7}2{0}
+{5}{$endif}{0}
+
+{4}// preprocessor2=6
+{6}(*$DEFINE xyz*){0}
+
+{4}// number=7
+{7}123{0}
+{7}1.23{0}
+{13}-{7}123{0}
+{13}-{7}12.3{0}
+{13}+{7}123{0}
+{7}123{0}
+{7}1.23e2{0}
+{13}-{7}1.23E2{0}
+
+{4}// hexnumber=8
+{8}$123{0}
+{8}$123ABCDEF{0}
+{8}$ABCDEF123{0}
+
+{4}// word=9
+{9}absolute{0} {9}abstract{0} {9}and{0} {9}array{0} {9}as{0} 
+
+{4}// string=10
+{10}'string'{0}
+
+{4}// stringeol=11
+{11}'string
+{0}
+{4}// character=12
+{12}#65{0}
+
+{4}// operator=13 
+{8}${0} {13}&{0} {13}*{0} {13}+{0} {13}/{0} {13}<{0} {13}={0} {13}>{0} {13}^{0}  
+
+{4}// asm
+{9}asm{14} 
+  this is 
+  inside assembler
+{9}end{0}

lexilla/test/examples/pascal/CodeFolding.pas

@@ -0,0 +1,80 @@
+// tests for code folding
+
+// multi line comments
+{ 
+line1
+line2
+}
+
+// begin .. end
+begin
+some commands
+end;
+
+record test
+  var1: type1;
+  var2: type2;
+  end; //record
+  
+//asm
+asm
+  some statement
+  end; //asm
+  
+//try (from https://wiki.freepascal.org/Try)
+try
+  // code that might generate an exception
+except
+  // will only be executed in case of an exception
+  on E: EDatabaseError do
+    ShowMessage( 'Database error: '+ E.ClassName + #13#10 + E.Message );
+  on E: Exception do
+    ShowMessage( 'Error: '+ E.ClassName + #13#10 + E.Message );
+end;
+  
+//try nested (from https://wiki.freepascal.org/Try)
+try
+  try
+    // code dealing with database that might generate an exception
+  except
+    // will only be executed in case of an exception
+    on E: EDatabaseError do
+      ShowMessage( 'Database error: '+ E.ClassName + #13#10 + E.Message );
+    on E: Exception do
+      ShowMessage( 'Error: '+ E.ClassName + #13#10 + E.Message );
+  end;
+finally
+  // clean up database-related resources
+end;
+
+//case
+case x of
+  1: do something;
+  2: do some other thing;
+else
+  do default;
+  end; //case
+  
+//if then else  
+if x=y then 
+  do something;
+else
+  do some other thing;
+  
+//for loop  
+for i:=1 to 10 do
+  writeln(i)
+
+//do until
+repeat
+  write(a);
+  i:=i+1;
+until i>10;
+
+//preprocessor if, else, endif
+{$DEFINE label}
+{$IFDEF label}
+  command 1
+{$ELSE}
+  command 2
+{$ENDIF}

lexilla/test/examples/pascal/CodeFolding.pas.folded

@@ -0,0 +1,81 @@
+ 0 400   0   // tests for code folding
+ 1 400   0   
+ 0 400   0   // multi line comments
+ 2 400   0 + { 
+ 0 401   0 | line1
+ 0 401   0 | line2
+ 0 401   0 | }
+ 1 400   0   
+ 0 400   0   // begin .. end
+ 2 400   0 + begin
+ 0 401   0 | some commands
+ 0 401   0 | end;
+ 1 400   0   
+ 2 400   0 + record test
+ 0 401   0 |   var1: type1;
+ 0 401   0 |   var2: type2;
+ 0 401   0 |   end; //record
+ 1 400   0     
+ 0 400   0   //asm
+ 2 400   0 + asm
+ 0 401   0 |   some statement
+ 0 401   0 |   end; //asm
+ 1 400   0     
+ 0 400   0   //try (from https://wiki.freepascal.org/Try)
+ 2 400   0 + try
+ 0 401   0 |   // code that might generate an exception
+ 0 401   0 | except
+ 0 401   0 |   // will only be executed in case of an exception
+ 0 401   0 |   on E: EDatabaseError do
+ 0 401   0 |     ShowMessage( 'Database error: '+ E.ClassName + #13#10 + E.Message );
+ 0 401   0 |   on E: Exception do
+ 0 401   0 |     ShowMessage( 'Error: '+ E.ClassName + #13#10 + E.Message );
+ 0 401   0 | end;
+ 1 400   0     
+ 0 400   0   //try nested (from https://wiki.freepascal.org/Try)
+ 2 400   0 + try
+ 2 401   0 +   try
+ 0 402   0 |     // code dealing with database that might generate an exception
+ 0 402   0 |   except
+ 0 402   0 |     // will only be executed in case of an exception
+ 0 402   0 |     on E: EDatabaseError do
+ 0 402   0 |       ShowMessage( 'Database error: '+ E.ClassName + #13#10 + E.Message );
+ 0 402   0 |     on E: Exception do
+ 0 402   0 |       ShowMessage( 'Error: '+ E.ClassName + #13#10 + E.Message );
+ 0 402   0 |   end;
+ 0 401   0 | finally
+ 0 401   0 |   // clean up database-related resources
+ 0 401   0 | end;
+ 1 400   0   
+ 0 400   0   //case
+ 2 400   0 + case x of
+ 0 401   0 |   1: do something;
+ 0 401   0 |   2: do some other thing;
+ 0 401   0 | else
+ 0 401   0 |   do default;
+ 0 401   0 |   end; //case
+ 1 400   0     
+ 0 400   0   //if then else  
+ 0 400   0   if x=y then 
+ 0 400   0     do something;
+ 0 400   0   else
+ 0 400   0     do some other thing;
+ 1 400   0     
+ 0 400   0   //for loop  
+ 0 400   0   for i:=1 to 10 do
+ 0 400   0     writeln(i)
+ 1 400   0   
+ 0 400   0   //do until
+ 0 400   0   repeat
+ 0 400   0     write(a);
+ 0 400   0     i:=i+1;
+ 0 400   0   until i>10;
+ 1 400   0   
+ 0 400   0   //preprocessor if, else, endif
+ 0 400   0   {$DEFINE label}
+ 2 400   0 + {$IFDEF label}
+ 0 401   0 |   command 1
+ 0 401   0 | {$ELSE}
+ 0 401   0 |   command 2
+ 0 401   0 | {$ENDIF}
+ 1 400   0   

lexilla/test/examples/pascal/CodeFolding.pas.styled

@@ -0,0 +1,80 @@
+{4}// tests for code folding
+{0}
+{4}// multi line comments
+{2}{ 
+line1
+line2
+}{0}
+
+{4}// begin .. end
+{9}begin{0}
+{1}some{0} {1}commands{0}
+{9}end{13};{0}
+
+{9}record{0} {1}test{0}
+  {1}var1{13}:{0} {1}type1{13};{0}
+  {1}var2{13}:{0} {1}type2{13};{0}
+  {9}end{13};{0} {4}//record
+{0}  
+{4}//asm
+{9}asm{14}
+  some statement
+  {9}end{13};{0} {4}//asm
+{0}  
+{4}//try (from https://wiki.freepascal.org/Try)
+{9}try{0}
+  {4}// code that might generate an exception
+{9}except{0}
+  {4}// will only be executed in case of an exception
+{0}  {9}on{0} {1}E{13}:{0} {1}EDatabaseError{0} {9}do{0}
+    {1}ShowMessage{13}({0} {10}'Database error: '{13}+{0} {1}E{13}.{1}ClassName{0} {13}+{0} {12}#13#10{0} {13}+{0} {1}E{13}.{9}Message{0} {13});{0}
+  {9}on{0} {1}E{13}:{0} {1}Exception{0} {9}do{0}
+    {1}ShowMessage{13}({0} {10}'Error: '{13}+{0} {1}E{13}.{1}ClassName{0} {13}+{0} {12}#13#10{0} {13}+{0} {1}E{13}.{9}Message{0} {13});{0}
+{9}end{13};{0}
+  
+{4}//try nested (from https://wiki.freepascal.org/Try)
+{9}try{0}
+  {9}try{0}
+    {4}// code dealing with database that might generate an exception
+{0}  {9}except{0}
+    {4}// will only be executed in case of an exception
+{0}    {9}on{0} {1}E{13}:{0} {1}EDatabaseError{0} {9}do{0}
+      {1}ShowMessage{13}({0} {10}'Database error: '{13}+{0} {1}E{13}.{1}ClassName{0} {13}+{0} {12}#13#10{0} {13}+{0} {1}E{13}.{9}Message{0} {13});{0}
+    {9}on{0} {1}E{13}:{0} {1}Exception{0} {9}do{0}
+      {1}ShowMessage{13}({0} {10}'Error: '{13}+{0} {1}E{13}.{1}ClassName{0} {13}+{0} {12}#13#10{0} {13}+{0} {1}E{13}.{9}Message{0} {13});{0}
+  {9}end{13};{0}
+{9}finally{0}
+  {4}// clean up database-related resources
+{9}end{13};{0}
+
+{4}//case
+{9}case{0} {1}x{0} {9}of{0}
+  {7}1{13}:{0} {9}do{0} {1}something{13};{0}
+  {7}2{13}:{0} {9}do{0} {1}some{0} {1}other{0} {1}thing{13};{0}
+{9}else{0}
+  {9}do{0} {1}default{13};{0}
+  {9}end{13};{0} {4}//case
+{0}  
+{4}//if then else  
+{9}if{0} {1}x{13}={1}y{0} {9}then{0} 
+  {9}do{0} {1}something{13};{0}
+{9}else{0}
+  {9}do{0} {1}some{0} {1}other{0} {1}thing{13};{0}
+  
+{4}//for loop  
+{9}for{0} {1}i{13}:={7}1{0} {9}to{0} {7}10{0} {9}do{0}
+  {1}writeln{13}({1}i{13}){0}
+
+{4}//do until
+{9}repeat{0}
+  {1}write{13}({1}a{13});{0}
+  {1}i{13}:={1}i{13}+{7}1{13};{0}
+{9}until{0} {1}i{13}>{7}10{13};{0}
+
+{4}//preprocessor if, else, endif
+{5}{$DEFINE label}{0}
+{5}{$IFDEF label}{0}
+  {1}command{0} {7}1{0}
+{5}{$ELSE}{0}
+  {1}command{0} {7}2{0}
+{5}{$ENDIF}{0}

lexilla/test/examples/pascal/SciTE.properties

@@ -0,0 +1,20 @@
+# coding: utf-8
+lexer.*.pas=pascal
+keywords.*.pas=absolute abstract and array as asm assembler automated begin case \
+cdecl class const constructor delayed deprecated destructor dispid dispinterface \
+div do downto dynamic else end except experimental export exports external far \
+file final finalization finally for forward function goto helper if \
+implementation in inherited initialization inline interface is label library \
+message mod near nil not object of on operator or out overload override packed \
+pascal platform private procedure program property protected public published \
+raise record reference register reintroduce repeat resourcestring safecall \
+sealed set shl shr static stdcall strict string then threadvar to try type unit \
+unsafe until uses var varargs virtual while winapi with xor
+
+
+lexer.pascal.smart.highlighting=1
+
+fold=1
+fold.preprocessor=1
+fold.comment=1
+fold.compact=1

lexilla/test/examples/pascal/SomeExample.pas

@@ -0,0 +1,41 @@
+// some example source code
+
+{
+  SCE_PAS_DEFAULT=0
+  SCE_PAS_IDENTIFIER=1
+  SCE_PAS_COMMENT=2
+  SCE_PAS_COMMENT2=3
+  SCE_PAS_COMMENTLINE=4
+  SCE_PAS_PREPROCESSOR=5
+  SCE_PAS_PREPROCESSOR2=6
+  SCE_PAS_NUMBER=7
+  SCE_PAS_HEXNUMBER=8
+  SCE_PAS_WORD=9
+  SCE_PAS_STRING=10
+  SCE_PAS_STRINGEOL=11
+  SCE_PAS_CHARACTER=12
+  SCE_PAS_OPERATOR=13
+  SCE_PAS_ASM=14
+}
+
+{ --------------------------------------------------------------------------- }
+function functionname(paramerter1: type1):result1;
+  var
+    i: LongInt;
+  begin
+  for i:=1 to 10 do
+    begin
+    writeln(i)
+    end;
+  result:=true;
+  end;
+{ --------------------------------------------------------------------------- }
+procedure procedurename(parameter2: type2);
+  var
+    i: LongInt;
+  begin
+  for i:=1 to 10 do
+    begin
+    writeln(i)
+    end;
+  end;

lexilla/test/examples/pascal/SomeExample.pas.folded

@@ -0,0 +1,42 @@
+ 0 400   0   // some example source code
+ 1 400   0   
+ 2 400   0 + {
+ 0 401   0 |   SCE_PAS_DEFAULT=0
+ 0 401   0 |   SCE_PAS_IDENTIFIER=1
+ 0 401   0 |   SCE_PAS_COMMENT=2
+ 0 401   0 |   SCE_PAS_COMMENT2=3
+ 0 401   0 |   SCE_PAS_COMMENTLINE=4
+ 0 401   0 |   SCE_PAS_PREPROCESSOR=5
+ 0 401   0 |   SCE_PAS_PREPROCESSOR2=6
+ 0 401   0 |   SCE_PAS_NUMBER=7
+ 0 401   0 |   SCE_PAS_HEXNUMBER=8
+ 0 401   0 |   SCE_PAS_WORD=9
+ 0 401   0 |   SCE_PAS_STRING=10
+ 0 401   0 |   SCE_PAS_STRINGEOL=11
+ 0 401   0 |   SCE_PAS_CHARACTER=12
+ 0 401   0 |   SCE_PAS_OPERATOR=13
+ 0 401   0 |   SCE_PAS_ASM=14
+ 0 401   0 | }
+ 1 400   0   
+ 0 400   0   { --------------------------------------------------------------------------- }
+ 0 400   0   function functionname(paramerter1: type1):result1;
+ 0 400   0     var
+ 0 400   0       i: LongInt;
+ 2 400   0 +   begin
+ 0 401   0 |   for i:=1 to 10 do
+ 2 401   0 +     begin
+ 0 402   0 |     writeln(i)
+ 0 402   0 |     end;
+ 0 401   0 |   result:=true;
+ 0 401   0 |   end;
+ 0 400   0   { --------------------------------------------------------------------------- }
+ 0 400   0   procedure procedurename(parameter2: type2);
+ 0 400   0     var
+ 0 400   0       i: LongInt;
+ 2 400   0 +   begin
+ 0 401   0 |   for i:=1 to 10 do
+ 2 401   0 +     begin
+ 0 402   0 |     writeln(i)
+ 0 402   0 |     end;
+ 0 401   0 |   end;
+ 1 400   0   

lexilla/test/examples/pascal/SomeExample.pas.styled

@@ -0,0 +1,41 @@
+{4}// some example source code
+{0}
+{2}{
+  SCE_PAS_DEFAULT=0
+  SCE_PAS_IDENTIFIER=1
+  SCE_PAS_COMMENT=2
+  SCE_PAS_COMMENT2=3
+  SCE_PAS_COMMENTLINE=4
+  SCE_PAS_PREPROCESSOR=5
+  SCE_PAS_PREPROCESSOR2=6
+  SCE_PAS_NUMBER=7
+  SCE_PAS_HEXNUMBER=8
+  SCE_PAS_WORD=9
+  SCE_PAS_STRING=10
+  SCE_PAS_STRINGEOL=11
+  SCE_PAS_CHARACTER=12
+  SCE_PAS_OPERATOR=13
+  SCE_PAS_ASM=14
+}{0}
+
+{2}{ --------------------------------------------------------------------------- }{0}
+{9}function{0} {1}functionname{13}({1}paramerter1{13}:{0} {1}type1{13}):{1}result1{13};{0}
+  {9}var{0}
+    {1}i{13}:{0} {1}LongInt{13};{0}
+  {9}begin{0}
+  {9}for{0} {1}i{13}:={7}1{0} {9}to{0} {7}10{0} {9}do{0}
+    {9}begin{0}
+    {1}writeln{13}({1}i{13}){0}
+    {9}end{13};{0}
+  {1}result{13}:={1}true{13};{0}
+  {9}end{13};{0}
+{2}{ --------------------------------------------------------------------------- }{0}
+{9}procedure{0} {1}procedurename{13}({1}parameter2{13}:{0} {1}type2{13});{0}
+  {9}var{0}
+    {1}i{13}:{0} {1}LongInt{13};{0}
+  {9}begin{0}
+  {9}for{0} {1}i{13}:={7}1{0} {9}to{0} {7}10{0} {9}do{0}
+    {9}begin{0}
+    {1}writeln{13}({1}i{13}){0}
+    {9}end{13};{0}
+  {9}end{13};{0}

lexilla/test/examples/python/attributes/attrib-decorator.py

@@ -0,0 +1,9 @@
+# issue#294 also pointed out that decorator attributes behaved differently
+#   for left-justified decorators vs indented decorators
+
+@decorator.attribute
+def foo():
+    @decorator.attribute
+    def bar():
+        pass
+    bar()

lexilla/test/examples/python/attributes/attrib-decorator.py.folded

@@ -0,0 +1,10 @@
+ 0 400   0   # issue#294 also pointed out that decorator attributes behaved differently
+ 0 400   0   #   for left-justified decorators vs indented decorators
+ 1 400   0   
+ 0 400   0   @decorator.attribute
+ 2 400   0 + def foo():
+ 0 404   0 |     @decorator.attribute
+ 2 404   0 +     def bar():
+ 0 408   0 |         pass
+ 0 404   0 |     bar()
+ 1 404   0 | 

lexilla/test/examples/python/attributes/attrib-decorator.py.styled

@@ -0,0 +1,9 @@
+{1}# issue#294 also pointed out that decorator attributes behaved differently{0}
+{1}#   for left-justified decorators vs indented decorators{0}
+
+{15}@decorator{10}.{15}attribute{0}
+{5}def{0} {9}foo{10}():{0}
+    {15}@decorator{10}.{15}attribute{0}
+    {5}def{0} {9}bar{10}():{0}
+        {5}pass{0}
+    {11}bar{10}(){0}

lexilla/test/examples/python/attributes/attrib-id.py

@@ -0,0 +1,18 @@
+varname = 1
+# identifier in first line was mis-styled as attribute when bug existed
+
+# left comment ends with period.  this was okay before bug.
+varname = 2
+
+x = 1 # comment after code ends with period. this failed when bug existed.
+varname = 3
+
+def dummy():
+    # indented comment ends with period.this failed when bug existed.
+    varname = 4
+
+x = 2 ## comment block is the same.
+varname = 5
+
+# per issue#294, identifiers were mis-styled as attributes when at beginning of file
+# and when after a non-left-justifed comment

lexilla/test/examples/python/attributes/attrib-id.py.folded

@@ -0,0 +1,19 @@
+ 0 400   0   varname = 1
+ 0 400   0   # identifier in first line was mis-styled as attribute when bug existed
+ 1 400   0   
+ 0 400   0   # left comment ends with period.  this was okay before bug.
+ 0 400   0   varname = 2
+ 1 400   0   
+ 0 400   0   x = 1 # comment after code ends with period. this failed when bug existed.
+ 0 400   0   varname = 3
+ 1 400   0   
+ 2 400   0 + def dummy():
+ 0 404   0 |     # indented comment ends with period.this failed when bug existed.
+ 0 404   0 |     varname = 4
+ 1 400   0   
+ 0 400   0   x = 2 ## comment block is the same.
+ 0 400   0   varname = 5
+ 1 400   0   
+ 0 400   0   # per issue#294, identifiers were mis-styled as attributes when at beginning of file
+ 0 400   0   # and when after a non-left-justifed comment
+ 1 400   0   

lexilla/test/examples/python/attributes/attrib-id.py.styled

@@ -0,0 +1,18 @@
+{11}varname{0} {10}={0} {2}1{0}
+{1}# identifier in first line was mis-styled as attribute when bug existed{0}
+
+{1}# left comment ends with period.  this was okay before bug.{0}
+{11}varname{0} {10}={0} {2}2{0}
+
+{11}x{0} {10}={0} {2}1{0} {1}# comment after code ends with period. this failed when bug existed.{0}
+{11}varname{0} {10}={0} {2}3{0}
+
+{5}def{0} {9}dummy{10}():{0}
+    {1}# indented comment ends with period.this failed when bug existed.{0}
+    {11}varname{0} {10}={0} {2}4{0}
+
+{11}x{0} {10}={0} {2}2{0} {12}## comment block is the same.{0}
+{11}varname{0} {10}={0} {2}5{0}
+
+{1}# per issue#294, identifiers were mis-styled as attributes when at beginning of file{0}
+{1}# and when after a non-left-justifed comment{0}

lexilla/test/examples/ruby/234HereDoc.rb

@@ -6,3 +6,10 @@ A中
 puts <<中
 #{1+2}
 中
+
+def STDERR::error(x) = puts(x)
+def STDERR.error(x) = puts(x)
+
+STDERR.error <<EOF
+STDERR heredoc
+EOF

lexilla/test/examples/ruby/234HereDoc.rb.folded

@@ -6,4 +6,11 @@
  2 400   0 + puts <<中
  0 401   0 | #{1+2}
  0 401   0 | 中
+ 1 400   0   
+ 0 400   0   def STDERR::error(x) = puts(x)
+ 0 400   0   def STDERR.error(x) = puts(x)
+ 1 400   0   
+ 2 400   0 + STDERR.error <<EOF
+ 0 401   0 | STDERR heredoc
+ 0 401   0 | EOF
  0 400   0   

lexilla/test/examples/ruby/234HereDoc.rb.styled

@@ -6,3 +6,10 @@
 {11}puts{0} {10}<<{20}中{22}
 {10}#{{4}1{10}+{4}2{10}}{22}
 {20}中{0}
+
+{5}def{0} {128}STDERR{10}::{9}error{10}({11}x{10}){0} {10}={0} {11}puts{10}({11}x{10}){0}
+{5}def{0} {128}STDERR{10}.{9}error{10}({11}x{10}){0} {10}={0} {11}puts{10}({11}x{10}){0}
+
+{128}STDERR{10}.{11}error{0} {10}<<{20}EOF{22}
+STDERR heredoc
+{20}EOF{0}

lexilla/test/examples/ruby/AllStyles.rb

@@ -113,6 +113,12 @@ while 1 do end
 #44:symbol
 %s(#{1 + 1})
 
+#128:user keywords 1
+decrypt
+
+#129:user keywords 2
+encrypt
+
 #19:data section
 __END__
 

lexilla/test/examples/ruby/AllStyles.rb.folded

@@ -113,6 +113,12 @@
  0 400   0   #44:symbol
  0 400   0   %s(#{1 + 1})
  1 400   0   
+ 0 400   0   #128:user keywords 1
+ 0 400   0   decrypt
+ 1 400   0   
+ 0 400   0   #129:user keywords 2
+ 0 400   0   encrypt
+ 1 400   0   
  0 400   0   #19:data section
  0 400   0   __END__
  1 400   0   

lexilla/test/examples/ruby/AllStyles.rb.styled

@@ -113,6 +113,12 @@
 {2}#44:symbol{0}
 {44}%s(#{1 + 1}){0}
 
+{2}#128:user keywords 1{0}
+{128}decrypt{0}
+
+{2}#129:user keywords 2{0}
+{129}encrypt{0}
+
 {2}#19:data section{0}
 {19}__END__
 

lexilla/test/examples/ruby/Issue65.rb

@@ -41,6 +41,7 @@ class A
 	# array decomposition
 	def dec(((a, b), c)) = puts(a + b + c)
 	# class method
+	def self::say(*s) = puts(s)
 	def self.say(*s) = puts(s)
 	# test short method name
 	def a = 1

lexilla/test/examples/ruby/Issue65.rb.folded

@@ -41,6 +41,7 @@
  0 401   0 | 	# array decomposition
  0 401   0 | 	def dec(((a, b), c)) = puts(a + b + c)
  0 401   0 | 	# class method
+ 0 401   0 | 	def self::say(*s) = puts(s)
  0 401   0 | 	def self.say(*s) = puts(s)
  0 401   0 | 	# test short method name
  0 401   0 | 	def a = 1

lexilla/test/examples/ruby/Issue65.rb.styled

@@ -41,6 +41,7 @@
 	{2}# array decomposition{0}
 	{5}def{0} {9}dec{10}((({11}a{10},{0} {11}b{10}),{0} {11}c{10})){0} {10}={0} {11}puts{10}({11}a{0} {10}+{0} {11}b{0} {10}+{0} {11}c{10}){0}
 	{2}# class method{0}
+	{5}def{0} {29}self{10}::{9}say{10}(*{11}s{10}){0} {10}={0} {11}puts{10}({11}s{10}){0}
 	{5}def{0} {29}self{10}.{9}say{10}(*{11}s{10}){0} {10}={0} {11}puts{10}({11}s{10}){0}
 	{2}# test short method name{0}
 	{5}def{0} {9}a{0} {10}={0} {4}1{0}

lexilla/test/examples/ruby/SciTE.properties

@@ -1,4 +1,8 @@
 lexer.*.rb=ruby
-keywords.*.rb=begin class def do end false if module return self super true unless until while \
+keywords.*.rb=begin class def do else end false if module return self super true unless until while \
 __FILE__ __LINE__
 fold=1
+
+substyles.ruby.11=2
+substylewords.11.1.*=decrypt STDERR
+substylewords.11.2.*=encrypt

lexilla/version.txt

@@ -1 +1 @@
-542
+543

scintilla/call/ScintillaCall.cxx

@@ -1259,6 +1259,26 @@ ChangeHistoryOption ScintillaCall::ChangeHistory() {
 	return static_cast<Scintilla::ChangeHistoryOption>(Call(Message::GetChangeHistory));
 }
 
+void ScintillaCall::SetUndoSelectionHistory(Scintilla::UndoSelectionHistoryOption undoSelectionHistory) {
+	Call(Message::SetUndoSelectionHistory, static_cast<uintptr_t>(undoSelectionHistory));
+}
+
+UndoSelectionHistoryOption ScintillaCall::UndoSelectionHistory() {
+	return static_cast<Scintilla::UndoSelectionHistoryOption>(Call(Message::GetUndoSelectionHistory));
+}
+
+void ScintillaCall::SetSelectionSerialized(const char *selectionString) {
+	CallString(Message::SetSelectionSerialized, 0, selectionString);
+}
+
+Position ScintillaCall::SelectionSerialized(char *selectionString) {
+	return CallPointer(Message::GetSelectionSerialized, 0, selectionString);
+}
+
+std::string ScintillaCall::SelectionSerialized() {
+	return CallReturnString(Message::GetSelectionSerialized, 0);
+}
+
 Line ScintillaCall::FirstVisibleLine() {
 	return Call(Message::GetFirstVisibleLine);
 }

scintilla/cocoa/Scintilla/Info.plist

@@ -15,7 +15,7 @@
 	<key>CFBundlePackageType</key>
 	<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
 	<key>CFBundleShortVersionString</key>
-	<string>5.5.4</string>
+	<string>5.5.5</string>
 	<key>CFBundleVersion</key>
 	<string>$(CURRENT_PROJECT_VERSION)</string>
 	<key>NSHumanReadableCopyright</key>

scintilla/cocoa/Scintilla/Scintilla.xcodeproj/project.pbxproj

@@ -586,7 +586,7 @@
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 5.5.4;
+				CURRENT_PROJECT_VERSION = 5.5.5;
 				DEAD_CODE_STRIPPING = YES;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
@@ -650,7 +650,7 @@
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 5.5.4;
+				CURRENT_PROJECT_VERSION = 5.5.5;
 				DEAD_CODE_STRIPPING = YES;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				ENABLE_NS_ASSERTIONS = NO;
@@ -682,7 +682,7 @@
 				CODE_SIGN_IDENTITY = "-";
 				CODE_SIGN_STYLE = Manual;
 				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 5.5.4;
+				CURRENT_PROJECT_VERSION = 5.5.5;
 				DEAD_CODE_STRIPPING = YES;
 				DEFINES_MODULE = YES;
 				DEVELOPMENT_TEAM = "";
@@ -717,7 +717,7 @@
 				CODE_SIGN_IDENTITY = "-";
 				CODE_SIGN_STYLE = Manual;
 				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 5.5.4;
+				CURRENT_PROJECT_VERSION = 5.5.5;
 				DEAD_CODE_STRIPPING = YES;
 				DEFINES_MODULE = YES;
 				DEVELOPMENT_TEAM = "";

scintilla/doc/ScintillaDoc.html

@@ -130,7 +130,7 @@
 
     <h1>Scintilla Documentation</h1>
 
-    <p>Last edited 26 October 2024 NH</p>
+    <p>Last edited 12 February 2025 NH</p>
 
     <p style="background:#90F0C0">Scintilla 5 has moved the lexers from Scintilla into a new
     <a href="Lexilla.html">Lexilla</a> project.<br />
@@ -1121,7 +1121,7 @@ struct Sci_TextRangeFull {
     If the text argument is NULL(0) then the length that should be allocated
     to store the entire current line is returned.</p>
 
-    <p>See also: <code
+    <p>See also: <code>
     <a class="seealso" href="#SCI_GETSELTEXT">SCI_GETSELTEXT</a>,
     <a class="seealso" href="#SCI_GETLINE">SCI_GETLINE</a>,
     <a class="seealso" href="#SCI_GETTEXT">SCI_GETTEXT</a>,
@@ -1239,6 +1239,10 @@ struct Sci_TextRangeFull {
      <a class="message" href="#SCI_GETRECTANGULARSELECTIONANCHORVIRTUALSPACE">SCI_GETRECTANGULARSELECTIONANCHORVIRTUALSPACE &rarr; position</a><br />
      <br />
 
+     <a class="message" href="#SCI_SETSELECTIONSERIALIZED">SCI_SETSELECTIONSERIALIZED(&lt;unused&gt;, const char *selectionString)</a><br />
+     <a class="message" href="#SCI_GETSELECTIONSERIALIZED">SCI_GETSELECTIONSERIALIZED(&lt;unused&gt;, char *selectionString) &rarr; position</a><br />
+     <br />
+
      <a class="element" href="#SC_ELEMENT_SELECTION_ADDITIONAL_TEXT">SC_ELEMENT_SELECTION_ADDITIONAL_TEXT : colouralpha</a><br />
      <a class="element" href="#SC_ELEMENT_SELECTION_ADDITIONAL_BACK">SC_ELEMENT_SELECTION_ADDITIONAL_BACK : colouralpha</a><br />
      <a class="discouraged message" href="#SCI_SETADDITIONALSELALPHA">SCI_SETADDITIONALSELALPHA(alpha alpha)</a><br />
@@ -1404,6 +1408,12 @@ struct Sci_TextRangeFull {
      Set or query the position and amount of virtual space for the caret and anchor of the rectangular selection.
      After setting the rectangular selection, this is broken down into multiple selections, one for each line.</p>
 
+    <p>
+     <b id="SCI_SETSELECTIONSERIALIZED">SCI_SETSELECTIONSERIALIZED(&lt;unused&gt;, const char *selectionString)</b><br />
+     <b id="SCI_GETSELECTIONSERIALIZED">SCI_GETSELECTIONSERIALIZED(&lt;unused&gt;, char *selectionString) &rarr; position</b><br />
+     Set or query the selection type and positions as a serialized string.
+     The format of this string may change in future versions so should not be persisted beyond the current session.</p>
+
     <p>
      <b id="SC_ELEMENT_SELECTION_ADDITIONAL_TEXT">SC_ELEMENT_SELECTION_ADDITIONAL_TEXT : colouralpha</b><br />
      <b id="SC_ELEMENT_SELECTION_ADDITIONAL_BACK">SC_ELEMENT_SELECTION_ADDITIONAL_BACK : colouralpha</b><br />
@@ -1938,6 +1948,8 @@ struct Sci_TextToFindFull {
      <a class="message" href="#SCI_ENDUNDOACTION">SCI_ENDUNDOACTION</a><br />
      <a class="message" href="#SCI_GETUNDOSEQUENCE">SCI_GETUNDOSEQUENCE &rarr; int</a><br />
      <a class="message" href="#SCI_ADDUNDOACTION">SCI_ADDUNDOACTION(int token, int flags)</a><br />
+     <a class="message" href="#SCI_SETUNDOSELECTIONHISTORY">SCI_SETUNDOSELECTIONHISTORY(int undoSelectionHistory)</a><br />
+     <a class="message" href="#SCI_GETUNDOSELECTIONHISTORY">SCI_GETUNDOSELECTIONHISTORY &rarr; int</a><br />
     </code>
 
     <p><b id="SCI_UNDO">SCI_UNDO</b><br />
@@ -2016,6 +2028,35 @@ struct Sci_TextToFindFull {
      look like typing or deletions that look like multiple uses of the Backspace or Delete keys.
      </p>
 
+    <p><b id="SCI_SETUNDOSELECTIONHISTORY">SCI_SETUNDOSELECTIONHISTORY(int undoSelectionHistory)</b><br />
+     <b id="SCI_GETUNDOSELECTIONHISTORY">SCI_GETUNDOSELECTIONHISTORY &rarr; int</b><br />
+     The selection for each action can be saved and then restored when undo or redo is performed.
+      <code>SCI_SETUNDOSELECTIONHISTORY</code> controls this.
+      There is a memory cost for this feature with a minimum of 150 bytes for each of undo and redo for each recorded action.
+      Recording may be turned on at any time.</p>
+
+    <p>The <code class="parameter">undoSelectionHistory</code> argument can be:</p>
+    <table class="standard" summary="Undo selection history state">
+      <tbody valign="top">
+        <tr>
+          <th align="left"><code>SC_UNDO_SELECTION_HISTORY_DISABLED</code></th>
+
+          <td>0</td>
+
+          <td>The default: undo selection history turned off.</td>
+        </tr>
+
+        <tr>
+          <th align="left"><code>SC_UNDO_SELECTION_HISTORY_ENABLED</code></th>
+
+          <td>1</td>
+
+          <td>Restore selection for each undo and redo.</td>
+        </tr>
+
+      </tbody>
+    </table>
+
     <h2 id="UndoSaveRestore">Undo Save and Restore</h2>
 
     <p>This feature is unfinished and has limitations.
@@ -5024,16 +5065,55 @@ struct Sci_TextToFindFull {
      <b id="SCI_GETTECHNOLOGY">SCI_GETTECHNOLOGY &rarr; int</b><br />
     The technology property allows choosing between different drawing APIs and options.
     On most platforms, the only choice is <code>SC_TECHNOLOGY_DEFAULT</code> (0).
-    On Windows Vista or later, <code>SC_TECHNOLOGY_DIRECTWRITE</code> (1),
-    <code>SC_TECHNOLOGY_DIRECTWRITERETAIN</code> (2), or
-    <code>SC_TECHNOLOGY_DIRECTWRITEDC</code> (3)
-    can be chosen to use the Direct2D and DirectWrite APIs for higher quality antialiased drawing.
-    <code>SC_TECHNOLOGY_DIRECTWRITERETAIN</code> differs from
-    <code>SC_TECHNOLOGY_DIRECTWRITE</code> by requesting that the frame
-    is retained after being presented which may prevent drawing failures on some cards and drivers.
-    <code>SC_TECHNOLOGY_DIRECTWRITEDC</code> differs from
-    <code>SC_TECHNOLOGY_DIRECTWRITE</code> by using DirectWrite to draw into a GDI DC.
+    On Windows 7 or later, <code>SC_TECHNOLOGY_DIRECTWRITE</code> (1) performs higher quality antialiased drawing.
+    The other choices in the following table are similar to <code>SC_TECHNOLOGY_DIRECTWRITE</code>
+    but may work or be more efficient in some situations where <code>SC_TECHNOLOGY_DIRECTWRITE</code> fails or is slow.
     </p>
+    <table class="standard" summary="Technology choice">
+      <tbody valign="top">
+        <tr>
+          <th align="left"><code>SC_TECHNOLOGY_DEFAULT</code></th>
+
+          <td>0</td>
+
+          <td>Use older GDI API which is compatible with all versions of Windows including Windows Vista and XP.</td>
+        </tr>
+
+        <tr>
+          <th align="left"><code>SC_TECHNOLOGY_DIRECTWRITE</code></th>
+
+          <td>1</td>
+
+          <td>Use the Direct2D and DirectWrite APIs for higher quality antialiased drawing.</td>
+        </tr>
+
+        <tr>
+          <th align="left"><code>SC_TECHNOLOGY_DIRECTWRITERETAIN</code></th>
+
+          <td>2</td>
+
+          <td>Request that the frame is retained after being presented which may prevent drawing failures on some cards and drivers.</td>
+        </tr>
+
+        <tr>
+          <th align="left"><code>SC_TECHNOLOGY_DIRECTWRITEDC</code></th>
+
+          <td>3</td>
+
+          <td>Use DirectWrite to draw into a GDI DC. This mode may work for remote access sessions.</td>
+        </tr>
+
+        <tr>
+          <th align="left"><code>SC_TECHNOLOGY_DIRECT_WRITE_1</code></th>
+
+          <td>4</td>
+
+          <td>Use DirectWrite in a lower level way that manages graphics state more explicitly.</td>
+        </tr>
+
+      </tbody>
+    </table>
+
     <p>
     On Win32, buffered drawing is set to a reasonable value for the technology: on for GDI and off for Direct2D
     as Direct2D performs its own buffering.
@@ -5216,7 +5296,7 @@ struct Sci_TextToFindFull {
           <td><code>SC_SUPPORTS_LINE_DRAWS_FINAL</code></td>
           <td>0</td>
           <td>Whether drawing a line draws its final position.<br />
-          Only false on Win32 GDI.</div></td>
+          Only false on Win32 GDI.</td>
         </tr>
 
         <tr>
@@ -7523,7 +7603,7 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){
     <p>For many applications lexing documents larger than 4GB will be too sluggish so
     <code>SC_DOCUMENTOPTION_STYLES_NONE</code>
     and the null lexer <code>"null"</code> can be used. Another approach is to turn on idle styling with
-    <code><a class="seealso" href="#SCI_SETIDLESTYLING">SCI_SETIDLESTYLING</code></a>.
+    <a class="seealso" href="#SCI_SETIDLESTYLING"><code>SCI_SETIDLESTYLING</code></a>.
     </p>
 
     <table class="standard" summary="Document options">

scintilla/doc/ScintillaDownload.html

@@ -26,9 +26,9 @@
     <table bgcolor="#CCCCCC" width="100%" cellspacing="0" cellpadding="8" border="0">
       <tr>
         <td>
-          <font size="4"> <a href="https://www.scintilla.org/scintilla554.zip">
+          <font size="4"> <a href="https://www.scintilla.org/scintilla555.zip">
 	Windows</a>&nbsp;&nbsp;
-	<a href="https://www.scintilla.org/scintilla554.tgz">
+	<a href="https://www.scintilla.org/scintilla555.tgz">
           GTK/Linux</a>&nbsp;&nbsp;
 	</font>
         </td>
@@ -42,7 +42,7 @@
       containing very few restrictions.
     </p>
     <h3>
-       Release 5.5.4
+       Release 5.5.5
     </h3>
     <h4>
        Source Code
@@ -50,8 +50,8 @@
        The source code package contains all of the source code for Scintilla but no binary
 	executable code and is available in
        <ul>
-       <li><a href="https://www.scintilla.org/scintilla554.zip">zip format</a> (1.8M) commonly used on Windows</li>
-       <li><a href="https://www.scintilla.org/scintilla554.tgz">tgz format</a> (1.7M) commonly used on Linux and compatible operating systems</li>
+       <li><a href="https://www.scintilla.org/scintilla555.zip">zip format</a> (1.8M) commonly used on Windows</li>
+       <li><a href="https://www.scintilla.org/scintilla555.tgz">tgz format</a> (1.7M) commonly used on Linux and compatible operating systems</li>
        </ul>
        Instructions for building on both Windows and Linux are included in the readme file.
     <h4>

scintilla/doc/ScintillaHistory.html

@@ -34,6 +34,7 @@
       </tr>
     </table>
     <h1>History of Scintilla</h1>
+    <a href="#Releases">Releases</a>
     <h2>Contributors</h2>
     <p>
        Thanks to all the people that have contributed patches, bug reports and suggestions.
@@ -586,7 +587,41 @@
 	<td>Pawel Z Wronek</td>
     </tr>
     </table>
-    <h2>Releases</h2>
+    <h2 id="Releases">Releases</h2>
+    <h3>
+       <a href="https://www.scintilla.org/scintilla555.zip">Release 5.5.5</a>
+    </h3>
+    <ul>
+	<li>
+	Released 25 February 2025.
+	</li>
+	<li>
+	Remember selection with undo and redo. Controlled with SCI_SETUNDOSELECTIONHISTORY.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1273/">Feature #1273</a>,
+	<a href="https://sourceforge.net/p/scintilla/bugs/1479/">Bug #1479</a>,
+	<a href="https://sourceforge.net/p/scintilla/bugs/1224/">Bug #1224</a>.
+	</li>
+	<li>
+	Serialize selection type and ranges with SCI_GETSELECTIONSERIALIZED and SCI_SETSELECTIONSERIALIZED.
+	</li>
+	<li>
+	For Win32, update Direct2D and DirectWrite interfaces used to 1.1 and add a lower-level approach to calling DirectWrite 1.1
+	by specifying SC_TECHNOLOGY_DIRECT_WRITE_1.
+	Since Windows Vista does not support these API versions, Scintilla o longer supports DirectWrite on Windows Vista and will
+	fall back to using GDI.
+	</li>
+	<li>
+	Fix segmentation of long lexemes to avoid breaking before modifiers like accents that must be drawn with their base letters.
+	For wrapping, try to break lines without separating letters from modifiers.
+	</li>
+	<li>
+	For GTK on Windows, replace reverse arrow cursor with hand as reverse arrow was small in scaled modes.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2460/">Bug #2460</a>.
+	</li>
+	<li>
+	Fix bug on Qt where double-click stopped working when Scintilla instance had been running for weeks.
+	</li>
+    </ul>
     <h3>
        <a href="https://www.scintilla.org/scintilla554.zip">Release 5.5.4</a>
     </h3>

scintilla/doc/index.html

@@ -9,7 +9,7 @@
     <meta name="keywords" content="Scintilla, SciTE, Editing Component, Text Editor" />
     <meta name="Description"
     content="www.scintilla.org is the home of the Scintilla editing component and SciTE text editor application." />
-    <meta name="Date.Modified" content="20241218" />
+    <meta name="Date.Modified" content="20250225" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <style type="text/css">
         .logo {
@@ -61,8 +61,8 @@
           GTK, and macOS</font>
         </td>
         <td width="40%" align="right">
-          <font color="#FFCC99" size="3"> Release version 5.5.4<br />
-           Site last modified December 18 2024</font>
+          <font color="#FFCC99" size="3"> Release version 5.5.5<br />
+           Site last modified February 25 2025</font>
         </td>
         <td width="20%">
           &nbsp;
@@ -77,11 +77,11 @@
       </tr>
     </table>
     <ul id="versionlist">
+      <li>Version 5.5.5 can remember selection with undo and redo. Update to use DirectWrite 1.1.</li>
       <li>Version 5.5.4 fixes wrapping of removed lines and edge cases for moving lines. On GTK, middle click can be repeated with one value.</li>
       <li>Version 5.5.3 fixes horizontal scrolling with a touchpad on Win32.</li>
       <li>Version 5.5.2 adds multiple selection copy with separator, a font stretch setting, and access to whether an undo sequence is active.</li>
       <li>Version 5.5.1 adds SCI_CUTALLOWLINE and fixes a Win32 bug that caused the cursor to flicker.</li>
-      <li>Version 5.5.0 adds elements for inactive additional selections.</li>
     </ul>
     <ul id="menu">
       <li id="remote1"><a href="https://www.scintilla.org/SciTEImage.html">Screenshot</a></li>

scintilla/gtk/PlatGTK.cxx

@@ -1346,7 +1346,13 @@ void Window::SetCursor(Cursor curs) {
 		gdkCurs = gdk_cursor_new_for_display(pdisplay, GDK_HAND2);
 		break;
 	case Cursor::reverseArrow:
+#ifdef G_OS_WIN32
+		// GDK_RIGHT_PTR is scaled incorrectly under Windows with HiDPI screens (GTK 3.24);
+		// GDK_HAND2 is mapped to a native Windows cursor by GTK
+		gdkCurs = gdk_cursor_new_for_display(pdisplay, GDK_HAND2);
+#else
 		gdkCurs = gdk_cursor_new_for_display(pdisplay, GDK_RIGHT_PTR);
+#endif
 		break;
 	default:
 		gdkCurs = gdk_cursor_new_for_display(pdisplay, GDK_LEFT_PTR);

scintilla/include/Scintilla.h

@@ -533,6 +533,12 @@ typedef sptr_t (*SciFnDirectStatus)(sptr_t ptr, unsigned int iMessage, uptr_t wP
 #define SC_CHANGE_HISTORY_INDICATORS 4
 #define SCI_SETCHANGEHISTORY 2780
 #define SCI_GETCHANGEHISTORY 2781
+#define SC_UNDO_SELECTION_HISTORY_DISABLED 0
+#define SC_UNDO_SELECTION_HISTORY_ENABLED 1
+#define SCI_SETUNDOSELECTIONHISTORY 2782
+#define SCI_GETUNDOSELECTIONHISTORY 2783
+#define SCI_SETSELECTIONSERIALIZED 2784
+#define SCI_GETSELECTIONSERIALIZED 2785
 #define SCI_GETFIRSTVISIBLELINE 2152
 #define SCI_GETLINE 2153
 #define SCI_GETLINECOUNT 2154
@@ -1122,6 +1128,7 @@ typedef sptr_t (*SciFnDirectStatus)(sptr_t ptr, unsigned int iMessage, uptr_t wP
 #define SC_TECHNOLOGY_DIRECTWRITE 1
 #define SC_TECHNOLOGY_DIRECTWRITERETAIN 2
 #define SC_TECHNOLOGY_DIRECTWRITEDC 3
+#define SC_TECHNOLOGY_DIRECT_WRITE_1 4
 #define SCI_SETTECHNOLOGY 2630
 #define SCI_GETTECHNOLOGY 2631
 #define SCI_CREATELOADER 2632

scintilla/include/Scintilla.iface

@@ -1334,6 +1334,22 @@ set void SetChangeHistory=2780(ChangeHistoryOption changeHistory,)
 # Report change history status.
 get ChangeHistoryOption GetChangeHistory=2781(,)
 
+enu UndoSelectionHistoryOption=SC_UNDO_SELECTION_HISTORY_
+val SC_UNDO_SELECTION_HISTORY_DISABLED=0
+val SC_UNDO_SELECTION_HISTORY_ENABLED=1
+
+# Enable or disable undo selection history.
+set void SetUndoSelectionHistory=2782(UndoSelectionHistoryOption undoSelectionHistory,)
+
+# Report undo selection history status.
+get UndoSelectionHistoryOption GetUndoSelectionHistory=2783(,)
+
+# Set selection from serialized form.
+set void SetSelectionSerialized=2784(, string selectionString)
+
+# Retrieve serialized form of selection.
+get position GetSelectionSerialized=2785(, stringresult selectionString)
+
 # Retrieve the display line at the top of the display.
 get line GetFirstVisibleLine=2152(,)
 
@@ -3076,6 +3092,7 @@ val SC_TECHNOLOGY_DEFAULT=0
 val SC_TECHNOLOGY_DIRECTWRITE=1
 val SC_TECHNOLOGY_DIRECTWRITERETAIN=2
 val SC_TECHNOLOGY_DIRECTWRITEDC=3
+val SC_TECHNOLOGY_DIRECT_WRITE_1=4
 
 ali SC_TECHNOLOGY_DIRECTWRITE=DIRECT_WRITE
 ali SC_TECHNOLOGY_DIRECTWRITERETAIN=DIRECT_WRITE_RETAIN

scintilla/include/ScintillaCall.h

@@ -360,6 +360,11 @@ public:
 	Position FormatRangeFull(bool draw, RangeToFormatFull *fr);
 	void SetChangeHistory(Scintilla::ChangeHistoryOption changeHistory);
 	Scintilla::ChangeHistoryOption ChangeHistory();
+	void SetUndoSelectionHistory(Scintilla::UndoSelectionHistoryOption undoSelectionHistory);
+	Scintilla::UndoSelectionHistoryOption UndoSelectionHistory();
+	void SetSelectionSerialized(const char *selectionString);
+	Position SelectionSerialized(char *selectionString);
+	std::string SelectionSerialized();
 	Line FirstVisibleLine();
 	Position GetLine(Line line, char *text);
 	std::string GetLine(Line line);

scintilla/include/ScintillaMessages.h

@@ -285,6 +285,10 @@ enum class Message {
 	FormatRangeFull = 2777,
 	SetChangeHistory = 2780,
 	GetChangeHistory = 2781,
+	SetUndoSelectionHistory = 2782,
+	GetUndoSelectionHistory = 2783,
+	SetSelectionSerialized = 2784,
+	GetSelectionSerialized = 2785,
 	GetFirstVisibleLine = 2152,
 	GetLine = 2153,
 	GetLineCount = 2154,

scintilla/include/ScintillaTypes.h

@@ -300,6 +300,11 @@ enum class ChangeHistoryOption {
 	Indicators = 4,
 };
 
+enum class UndoSelectionHistoryOption {
+	Disabled = 0,
+	Enabled = 1,
+};
+
 enum class FoldLevel {
 	None = 0x0,
 	Base = 0x400,
@@ -511,6 +516,7 @@ enum class Technology {
 	DirectWrite = 1,
 	DirectWriteRetain = 2,
 	DirectWriteDC = 3,
+	DirectWrite1 = 4,
 };
 
 enum class LineEndType {

scintilla/qt/ScintillaEdit/ScintillaDocument.cpp

@@ -55,6 +55,7 @@ public:
     void NotifyDeleted(Document *doc, void *userData) noexcept override;
     void NotifyStyleNeeded(Document *doc, void *userData, Sci::Position endPos) override;
     void NotifyErrorOccurred(Document *doc, void *userData, Status status) override;
+    void NotifyGroupCompleted(Document *doc, void *userData) noexcept override;
 };
 
 WatcherHelper::WatcherHelper(ScintillaDocument *owner_) : owner(owner_) {
@@ -88,6 +89,10 @@ void WatcherHelper::NotifyErrorOccurred(Document *, void *, Status status) {
     emit owner->error_occurred(static_cast<int>(status));
 }
 
+void WatcherHelper::NotifyGroupCompleted(Document *, void *) noexcept {
+    // Needed to satisfy protocol. May implement an event in future.
+}
+
 ScintillaDocument::ScintillaDocument(QObject *parent, void *pdoc_) :
     QObject(parent), pdoc(static_cast<Scintilla::IDocumentEditable *>(pdoc_)), docWatcher(nullptr) {
     if (!pdoc) {

scintilla/qt/ScintillaEdit/ScintillaEdit.pro

@@ -13,7 +13,7 @@ TEMPLATE = lib
 CONFIG += lib_bundle
 CONFIG += c++1z
 
-VERSION = 5.5.4
+VERSION = 5.5.5
 
 SOURCES += \
     ScintillaEdit.cpp \

scintilla/qt/ScintillaEditBase/ScintillaEditBase.cpp

@@ -241,12 +241,12 @@ void ScintillaEditBase::keyPressEvent(QKeyEvent *event)
 		default:                    key = event->key(); break;
 	}
 
-	bool shift = QApplication::keyboardModifiers() & Qt::ShiftModifier;
-	bool ctrl  = QApplication::keyboardModifiers() & Qt::ControlModifier;
-	bool alt   = QApplication::keyboardModifiers() & Qt::AltModifier;
+	const bool shift = QApplication::keyboardModifiers() & Qt::ShiftModifier;
+	const bool ctrl  = QApplication::keyboardModifiers() & Qt::ControlModifier;
+	const bool alt   = QApplication::keyboardModifiers() & Qt::AltModifier;
 
 	bool consumed = false;
-	bool added = sqt->KeyDownWithModifiers(static_cast<Keys>(key),
+	const bool added = sqt->KeyDownWithModifiers(static_cast<Keys>(key),
 					       ModifierFlags(shift, ctrl, alt),
 					       &consumed) != 0;
 	if (!consumed)
@@ -283,7 +283,9 @@ void ScintillaEditBase::keyPressEvent(QKeyEvent *event)
 	emit keyPressed(event);
 }
 
-static int modifierTranslated(int sciModifier)
+namespace {
+
+int modifierTranslated(int sciModifier)
 {
 	switch (sciModifier) {
 		case SCMOD_SHIFT:
@@ -299,15 +301,25 @@ static int modifierTranslated(int sciModifier)
 	}
 }
 
+// Convert QElapsedTimer timestamp qint64 into unsigned int milliseconds wanted by Editor methods.
+unsigned int TimeOfEvent(const QElapsedTimer &timer)
+{
+	// Wrap every 2 billion milliseconds -> 24 days
+	constexpr qint64 maxTime = 2'000'000'000;
+	return static_cast<unsigned int>(timer.elapsed() % maxTime);
+}
+
+}
+
 void ScintillaEditBase::mousePressEvent(QMouseEvent *event)
 {
-	Point pos = PointFromQPoint(event->pos());
+	const Point pos = PointFromQPoint(event->pos());
 
 	emit buttonPressed(event);
 
 	if (event->button() == Qt::MiddleButton &&
 	    QApplication::clipboard()->supportsSelection()) {
-		SelectionPosition selPos = sqt->SPositionFromLocation(
+		const SelectionPosition selPos = sqt->SPositionFromLocation(
 					pos, false, false, sqt->UserVirtualSpace());
 		sqt->sel.Clear();
 		sqt->SetSelection(selPos, selPos);
@@ -316,15 +328,15 @@ void ScintillaEditBase::mousePressEvent(QMouseEvent *event)
 	}
 
 	if (event->button() == Qt::LeftButton) {
-		bool shift = QApplication::keyboardModifiers() & Qt::ShiftModifier;
-		bool ctrl  = QApplication::keyboardModifiers() & Qt::ControlModifier;
-		bool alt   = QApplication::keyboardModifiers() & modifierTranslated(sqt->rectangularSelectionModifier);
+		const bool shift = QApplication::keyboardModifiers() & Qt::ShiftModifier;
+		const bool ctrl  = QApplication::keyboardModifiers() & Qt::ControlModifier;
+		const bool alt   = QApplication::keyboardModifiers() & modifierTranslated(sqt->rectangularSelectionModifier);
 
-		sqt->ButtonDownWithModifiers(pos, time.elapsed(), ModifierFlags(shift, ctrl, alt));
+		sqt->ButtonDownWithModifiers(pos, TimeOfEvent(time), ModifierFlags(shift, ctrl, alt));
 	}
 
 	if (event->button() == Qt::RightButton) {
-		sqt->RightButtonDownWithModifiers(pos, time.elapsed(), ModifiersOfKeyboard());
+		sqt->RightButtonDownWithModifiers(pos, TimeOfEvent(time), ModifiersOfKeyboard());
 	}
 }
 
@@ -332,11 +344,11 @@ void ScintillaEditBase::mouseReleaseEvent(QMouseEvent *event)
 {
 	const QPoint point = event->pos();
 	if (event->button() == Qt::LeftButton)
-		sqt->ButtonUpWithModifiers(PointFromQPoint(point), time.elapsed(), ModifiersOfKeyboard());
+		sqt->ButtonUpWithModifiers(PointFromQPoint(point), TimeOfEvent(time), ModifiersOfKeyboard());
 
 	const sptr_t pos = send(SCI_POSITIONFROMPOINT, point.x(), point.y());
 	const sptr_t line = send(SCI_LINEFROMPOSITION, pos);
-	int modifiers = QApplication::keyboardModifiers();
+	const int modifiers = QApplication::keyboardModifiers();
 
 	emit textAreaClicked(line, modifiers);
 	emit buttonReleased(event);
@@ -350,21 +362,21 @@ void ScintillaEditBase::mouseDoubleClickEvent(QMouseEvent *event)
 
 void ScintillaEditBase::mouseMoveEvent(QMouseEvent *event)
 {
-	Point pos = PointFromQPoint(event->pos());
+	const Point pos = PointFromQPoint(event->pos());
 
-	bool shift = QApplication::keyboardModifiers() & Qt::ShiftModifier;
-	bool ctrl  = QApplication::keyboardModifiers() & Qt::ControlModifier;
-	bool alt   = QApplication::keyboardModifiers() & modifierTranslated(sqt->rectangularSelectionModifier);
+	const bool shift = QApplication::keyboardModifiers() & Qt::ShiftModifier;
+	const bool ctrl  = QApplication::keyboardModifiers() & Qt::ControlModifier;
+	const bool alt   = QApplication::keyboardModifiers() & modifierTranslated(sqt->rectangularSelectionModifier);
 
 	const KeyMod modifiers = ModifierFlags(shift, ctrl, alt);
 
-	sqt->ButtonMoveWithModifiers(pos, time.elapsed(), modifiers);
+	sqt->ButtonMoveWithModifiers(pos, TimeOfEvent(time), modifiers);
 }
 
 void ScintillaEditBase::contextMenuEvent(QContextMenuEvent *event)
 {
-	Point pos = PointFromQPoint(event->globalPos());
-	Point pt = PointFromQPoint(event->pos());
+	const Point pos = PointFromQPoint(event->globalPos());
+	const Point pt = PointFromQPoint(event->pos());
 	if (!sqt->PointInSelection(pt)) {
 		sqt->SetEmptySelection(sqt->PositionFromLocation(pt));
 	}
@@ -383,7 +395,7 @@ void ScintillaEditBase::dragEnterEvent(QDragEnterEvent *event)
 	} else if (event->mimeData()->hasText()) {
 		event->acceptProposedAction();
 
-		Point point = PointFromQPoint(event->pos());
+		const Point point = PointFromQPoint(event->pos());
 		sqt->DragEnter(point);
 	} else {
 		event->ignore();
@@ -402,7 +414,7 @@ void ScintillaEditBase::dragMoveEvent(QDragMoveEvent *event)
 	} else if (event->mimeData()->hasText()) {
 		event->acceptProposedAction();
 
-		Point point = PointFromQPoint(event->pos());
+		const Point point = PointFromQPoint(event->pos());
 		sqt->DragMove(point);
 	} else {
 		event->ignore();
@@ -417,8 +429,8 @@ void ScintillaEditBase::dropEvent(QDropEvent *event)
 	} else if (event->mimeData()->hasText()) {
 		event->acceptProposedAction();
 
-		Point point = PointFromQPoint(event->pos());
-		bool move = (event->source() == this &&
+		const Point point = PointFromQPoint(event->pos());
+		const bool move = (event->source() == this &&
                  event->proposedAction() == Qt::MoveAction);
 		sqt->Drop(point, event->mimeData(), move);
 	} else {
@@ -428,7 +440,7 @@ void ScintillaEditBase::dropEvent(QDropEvent *event)
 
 bool ScintillaEditBase::IsHangul(const QChar qchar)
 {
-	unsigned int unicode = qchar.unicode();
+	const unsigned int unicode = qchar.unicode();
 	// Korean character ranges used for preedit chars.
 	// http://www.programminginkorean.com/programming/hangul-in-unicode/
 	const bool HangulJamo = (0x1100 <= unicode && unicode <= 0x11FF);
@@ -465,22 +477,24 @@ void ScintillaEditBase::DrawImeIndicator(int indicator, int len)
 	}
 }
 
-static int GetImeCaretPos(QInputMethodEvent *event)
+namespace {
+
+int GetImeCaretPos(QInputMethodEvent *event)
 {
-	foreach (QInputMethodEvent::Attribute attr, event->attributes()) {
+	foreach (const QInputMethodEvent::Attribute attr, event->attributes()) {
 		if (attr.type == QInputMethodEvent::Cursor)
 			return attr.start;
 	}
 	return 0;
 }
 
-static std::vector<int> MapImeIndicators(QInputMethodEvent *event)
+std::vector<int> MapImeIndicators(QInputMethodEvent *event)
 {
 	std::vector<int> imeIndicator(event->preeditString().size(), IndicatorUnknown);
-	foreach (QInputMethodEvent::Attribute attr, event->attributes()) {
+	foreach (const QInputMethodEvent::Attribute attr, event->attributes()) {
 		if (attr.type == QInputMethodEvent::TextFormat) {
-			QTextFormat format = attr.value.value<QTextFormat>();
-			QTextCharFormat charFormat = format.toCharFormat();
+			const QTextFormat format = attr.value.value<QTextFormat>();
+			const QTextCharFormat charFormat = format.toCharFormat();
 
 			int indicator = IndicatorUnknown;
 			switch (charFormat.underlineStyle()) {
@@ -520,6 +534,8 @@ static std::vector<int> MapImeIndicators(QInputMethodEvent *event)
 	return imeIndicator;
 }
 
+}
+
 void ScintillaEditBase::inputMethodEvent(QInputMethodEvent *event)
 {
 	// Copy & paste by johnsonj with a lot of helps of Neil
@@ -596,8 +612,8 @@ void ScintillaEditBase::inputMethodEvent(QInputMethodEvent *event)
 		}
 
 		// Move IME carets.
-		int imeCaretPos = GetImeCaretPos(event);
-		int imeEndToImeCaretU16 = imeCaretPos - preeditStrLen;
+		const int imeCaretPos = GetImeCaretPos(event);
+		const int imeEndToImeCaretU16 = imeCaretPos - preeditStrLen;
 		const Sci::Position imeCaretPosDoc = sqt->pdoc->GetRelativePositionUTF16(sqt->CurrentPosition(), imeEndToImeCaretU16);
 
 		MoveImeCarets(- sqt->CurrentPosition() + imeCaretPosDoc);
@@ -671,7 +687,9 @@ QVariant ScintillaEditBase::inputMethodQuery(Qt::InputMethodQuery query) const
 
 		case Qt::ImCurrentSelection:
 		{
-			QVarLengthArray<char,1024> buffer(send(SCI_GETSELTEXT)+1);
+			// Most of the time selection is small so can be stack allocated
+			constexpr int reasonableSelectionLength = 1024;
+			QVarLengthArray<char,reasonableSelectionLength> buffer(send(SCI_GETSELTEXT)+1);
 			sends(SCI_GETSELTEXT, 0, buffer.data());
 
 			return sqt->StringFromDocument(buffer.constData());
@@ -727,7 +745,7 @@ void ScintillaEditBase::notifyParent(NotificationData scn)
 			const bool deleted = FlagSet(scn.modificationType, ModificationFlags::DeleteText);
 
 			const Scintilla::Position length = send(SCI_GETTEXTLENGTH);
-			bool firstLineAdded = (added && length == 1) ||
+			const bool firstLineAdded = (added && length == 1) ||
 			                      (deleted && length == 0);
 
 			if (scn.linesAdded != 0) {

scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro

@@ -13,7 +13,7 @@ TEMPLATE = lib
 CONFIG += lib_bundle
 CONFIG += c++1z
 
-VERSION = 5.5.4
+VERSION = 5.5.5
 
 SOURCES += \
     PlatQt.cpp \

scintilla/scripts/HeaderOrder.txt

@@ -52,6 +52,7 @@
 #include <memory>
 #include <numeric>
 #include <chrono>
+#include <charconv>
 #include <regex>
 #include <iostream>
 #include <sstream>
@@ -79,8 +80,10 @@
 #include <shellscalingapi.h>
 #include <zmouse.h>
 #include <ole2.h>
-#include <d2d1.h>
-#include <dwrite.h>
+#include <wrl.h>
+#include <d2d1_1.h>
+#include <d3d11_1.h>
+#include <dwrite_1.h>
 
 // Cocoa headers
 #include <Cocoa/Cocoa.h>

scintilla/src/CaseConvert.cxx

@@ -749,7 +749,7 @@ void CaseConverter::SetupConversions(CaseConversion conversion) {
 			break;
 		}
 		if (!converted.empty()) {
-			const int character = UnicodeFromUTF8(reinterpret_cast<const unsigned char *>(originUTF8.data()));
+			const int character = UnicodeFromUTF8(originUTF8);
 			Add(character, converted);
 		}
 	}

scintilla/src/CellBuffer.cxx

@@ -1075,6 +1075,10 @@ int CellBuffer::UndoSequenceDepth() const noexcept {
 	return uh->UndoSequenceDepth();
 }
 
+bool CellBuffer::AfterUndoSequenceStart() const noexcept {
+	return uh->AfterUndoSequenceStart();
+}
+
 void CellBuffer::AddUndoAction(Sci::Position token, bool mayCoalesce) {
 	bool startSequence = false;
 	uh->AppendAction(ActionType::container, token, nullptr, 0, startSequence, mayCoalesce);

scintilla/src/CellBuffer.h

@@ -167,6 +167,7 @@ public:
 	void BeginUndoAction(bool mayCoalesce=false) noexcept;
 	void EndUndoAction() noexcept;
 	int UndoSequenceDepth() const noexcept;
+	bool AfterUndoSequenceStart() const noexcept;
 	void AddUndoAction(Sci::Position token, bool mayCoalesce);
 	void DeleteUndoHistory() noexcept;
 

scintilla/src/Document.cxx

@@ -18,6 +18,7 @@
 #include <string_view>
 #include <vector>
 #include <array>
+#include <map>
 #include <forward_list>
 #include <optional>
 #include <algorithm>
@@ -132,6 +133,9 @@ size_t ActionDuration::ActionsInAllowedTime(double secondsAllowed) const noexcep
 	return std::lround(secondsAllowed / Duration());
 }
 
+const CharacterExtracted characterEmpty(unicodeReplacementChar, 0);
+const CharacterExtracted characterBadByte(unicodeReplacementChar, 1);
+
 CharacterExtracted::CharacterExtracted(const unsigned char *charBytes, size_t widthCharBytes) noexcept {
 	const int utf8status = UTF8Classify(charBytes, widthCharBytes);
 	if (utf8status & UTF8MaskInvalid) {
@@ -959,7 +963,7 @@ bool Document::NextCharacter(Sci::Position &pos, int moveDir) const noexcept {
 
 CharacterExtracted Document::CharacterAfter(Sci::Position position) const noexcept {
 	if (position >= LengthNoExcept()) {
-		return CharacterExtracted(unicodeReplacementChar, 0);
+		return characterEmpty;
 	}
 	const unsigned char leadByte = cb.UCharAt(position);
 	if (!dbcsCodePage || UTF8IsAscii(leadByte)) {
@@ -985,7 +989,7 @@ CharacterExtracted Document::CharacterAfter(Sci::Position position) const noexce
 
 CharacterExtracted Document::CharacterBefore(Sci::Position position) const noexcept {
 	if (position <= 0) {
-		return CharacterExtracted(unicodeReplacementChar, 0);
+		return characterEmpty;
 	}
 	const unsigned char previousByte = cb.UCharAt(position - 1);
 	if (0 == dbcsCodePage) {
@@ -1010,7 +1014,7 @@ CharacterExtracted Document::CharacterBefore(Sci::Position position) const noexc
 			}
 			// Else invalid UTF-8 so return position of isolated trail byte
 		}
-		return CharacterExtracted(unicodeReplacementChar, 1);
+		return characterBadByte;
 	} else {
 		// Moving backwards in DBCS is complex so use NextPosition
 		const Sci::Position posStartCharacter = NextPosition(position, -1);
@@ -1201,6 +1205,104 @@ bool Document::IsDBCSDualByteAt(Sci::Position pos) const noexcept {
 		&& IsDBCSTrailByteNoExcept(cb.CharAt(pos + 1));
 }
 
+namespace {
+
+// Remove any extra bytes after the last valid character.
+void DiscardEndFragment(std::string_view &text) noexcept {
+	if (!text.empty()) {
+		if (UTF8IsFirstByte(text.back())) {
+			// Ending with start of character byte is invalid
+			text.remove_suffix(1);
+		} else if (UTF8IsTrailByte(text.back())) {
+			// go back to the start of last character.
+			const size_t maxTrail = std::max<size_t>(UTF8MaxBytes - 1, text.length());
+			size_t trail = 1;
+			while (trail < maxTrail && UTF8IsTrailByte(text[text.length() - trail])) {
+				trail++;
+			}
+			const std::string_view endPortion = text.substr(text.length() - trail);
+			if (!UTF8IsValid(endPortion)) {
+				text.remove_suffix(trail);
+			}
+		}
+	}
+}
+
+constexpr bool IsBaseOfGrapheme(CharacterCategory cc) {
+	// \p{L}\p{N}\p{P}\p{Sm}\p{Sc}\p{So}\p{Zs}
+	switch (cc) {
+	case ccLu:
+	case ccLl:
+	case ccLt:
+	case ccLm:
+	case ccLo:
+	case ccNd:
+	case ccNl:
+	case ccNo:
+	case ccPc:
+	case ccPd:
+	case ccPs:
+	case ccPe:
+	case ccPi:
+	case ccPf:
+	case ccPo:
+	case ccSm:
+	case ccSc:
+	case ccSo:
+	case ccZs:
+		return true;
+	default:
+		// ccMn, ccMc, ccMe,
+		// ccSk,
+		// ccZl, ccZp,
+		// ccCc, ccCf, ccCs, ccCo, ccCn
+		return false;
+	}
+}
+
+CharacterExtracted LastCharacter(std::string_view text) noexcept {
+	if (text.empty())
+		return characterEmpty;
+	const size_t length = text.length();
+	size_t trail = length;
+	while ((trail > 0) && (length - trail < UTF8MaxBytes) && UTF8IsTrailByte(text[trail - 1]))
+		trail--;
+	const size_t start = (trail > 0) ? trail - 1 : trail;
+	const int utf8status = UTF8Classify(text.substr(start));
+	if (utf8status & UTF8MaskInvalid) {
+		return characterBadByte;
+	}
+	return { static_cast<unsigned int>(UnicodeFromUTF8(text.substr(start))),
+		static_cast<unsigned int>(utf8status & UTF8MaskWidth) };
+}
+
+}
+
+bool Scintilla::Internal::DiscardLastCombinedCharacter(std::string_view &text) noexcept {
+	// Handle the simple common case where a base character may be followed by
+	// accents and similar marks by discarding until start of base character.
+	// 
+	// From Grapheme_Cluster_Boundaries
+	// combining character sequence = ccs-base? ccs-extend+
+	// ccs-base := [\p{L}\p{N}\p{P}\p{S}\p{Zs}]
+	// ccs-extend := [\p{M}\p{Join_Control}]
+	// Modified to move Sk (Symbol Modifier) from ccs-base to ccs-extend to preserve modified emoji
+
+	std::string_view truncated = text;
+	while (truncated.length() > UTF8MaxBytes) {
+		// Give up when short
+		const CharacterExtracted ce = LastCharacter(truncated);
+		const CharacterCategory cc = CategoriseCharacter(static_cast<int>(ce.character));
+		truncated.remove_suffix(ce.widthBytes);
+		if (IsBaseOfGrapheme(cc)) {
+			text = truncated;
+			return true;
+		}
+	}
+	// No base character found so just leave as is
+	return false;
+}
+
 // Need to break text into segments near end but taking into account the
 // encoding to not break inside a UTF-8 or DBCS character and also trying
 // to avoid breaking inside a pair of combining characters, or inside
@@ -1214,7 +1316,8 @@ bool Document::IsDBCSDualByteAt(Sci::Position pos) const noexcept {
 // In preference order from best to worst:
 //   1) Break before or after spaces or controls
 //   2) Break at word and punctuation boundary for better kerning and ligature support
-//   3) Break after whole character, this may break combining characters
+//   3) Break before letter in UTF-8 to avoid breaking combining characters
+//   4) Break after whole character, this may break combining characters
 
 size_t Document::SafeSegment(std::string_view text) const noexcept {
 	// check space first as most written language use spaces.
@@ -1225,6 +1328,13 @@ size_t Document::SafeSegment(std::string_view text) const noexcept {
 	}
 
 	if (!dbcsCodePage || dbcsCodePage == CpUtf8) {
+		if (dbcsCodePage) {
+			// UTF-8
+			DiscardEndFragment(text);
+			if (DiscardLastCombinedCharacter(text)) {
+				return text.length();
+			}
+		}
 		// backward iterate for UTF-8 and single byte encoding to find word and punctuation boundary.
 		std::string_view::iterator it = text.end() - 1;
 		const bool punctuation = IsPunctuation(*it);
@@ -1235,14 +1345,7 @@ size_t Document::SafeSegment(std::string_view text) const noexcept {
 			}
 		} while (it != text.begin());
 
-		it = text.end() - 1;
-		if (dbcsCodePage) {
-			// for UTF-8 go back to the start of last character.
-			for (int trail = 0; trail < UTF8MaxBytes - 1 && UTF8IsTrailByte(*it); trail++) {
-				--it;
-			}
-		}
-		return it - text.begin();
+		return text.length() - 1;
 	}
 
 	{
@@ -1320,6 +1423,10 @@ bool Document::DeleteChars(Sci::Position pos, Sci::Position len) {
 	} else {
 		enteredModification++;
 		if (!cb.IsReadOnly()) {
+			if (cb.IsCollectingUndo() && cb.CanRedo()) {
+				// Abandoning some undo actions so truncate any later selections
+				TruncateUndoComments(cb.UndoCurrent());
+			}
 			NotifyModified(
 			    DocModification(
 			        ModificationFlags::BeforeDelete | ModificationFlags::User,
@@ -1373,6 +1480,10 @@ Sci::Position Document::InsertString(Sci::Position position, const char *s, Sci:
 		s = insertion.c_str();
 		insertLength = insertion.length();
 	}
+	if (cb.IsCollectingUndo() && cb.CanRedo()) {
+		// Abandoning some undo actions so truncate any later selections
+		TruncateUndoComments(cb.UndoCurrent());
+	}
 	NotifyModified(
 		DocModification(
 			ModificationFlags::BeforeInsert | ModificationFlags::User,
@@ -1556,6 +1667,20 @@ Sci::Position Document::Redo() {
 	return newPos;
 }
 
+void Document::EndUndoAction() noexcept {
+	cb.EndUndoAction();
+	if (UndoSequenceDepth() == 0) {
+		// Broadcast notification to views to allow end of group processing.
+		// NotifyGroupCompleted may throw (for memory exhaustion) but this method
+		// may not as it is called in UndoGroup destructor so ignore exception.
+		try {
+			NotifyGroupCompleted();
+		} catch (...) {
+			// Ignore any exception
+		}
+	}
+}
+
 int Document::UndoSequenceDepth() const noexcept {
 	return cb.UndoSequenceDepth();
 }
@@ -2389,11 +2514,11 @@ LineCharacterIndexType Document::LineCharacterIndex() const noexcept {
 }
 
 void Document::AllocateLineCharacterIndex(LineCharacterIndexType lineCharacterIndex) {
-	return cb.AllocateLineCharacterIndex(lineCharacterIndex);
+	cb.AllocateLineCharacterIndex(lineCharacterIndex);
 }
 
 void Document::ReleaseLineCharacterIndex(LineCharacterIndexType lineCharacterIndex) {
-	return cb.ReleaseLineCharacterIndex(lineCharacterIndex);
+	cb.ReleaseLineCharacterIndex(lineCharacterIndex);
 }
 
 Sci::Line Document::LinesTotal() const noexcept {
@@ -2504,6 +2629,32 @@ void Document::SetLexInterface(std::unique_ptr<LexInterface> pLexInterface) noex
 	pli = std::move(pLexInterface);
 }
 
+void Document::SetViewState(void *view, ViewStateShared pVSS) {
+	if (pVSS) {
+		viewData[view] = pVSS;
+	} else {
+		viewData.erase(view);
+	}
+}
+
+ViewStateShared Document::GetViewState(void *view) const noexcept {
+	try {
+		const std::map<void *, ViewStateShared>::const_iterator it = viewData.find(view);
+
+		if (it != viewData.end()) {
+			return it->second;
+		}
+	} catch (...) {
+	}
+	return {};
+}
+
+void Document::TruncateUndoComments(int action) {
+	for (auto &[key, value] : viewData) {
+		value->TruncateUndo(action);
+	}
+}
+
 int SCI_METHOD Document::SetLineState(Sci_Position line, int state) {
 	const int statePrevious = States()->SetLineState(line, state, LinesTotal());
 	if (state != statePrevious) {
@@ -2700,6 +2851,12 @@ void Document::NotifySavePoint(bool atSavePoint) {
 	}
 }
 
+void Document::NotifyGroupCompleted() noexcept {
+	for (const WatcherWithUserData &watcher : watchers) {
+		watcher.watcher->NotifyGroupCompleted(this, watcher.userData);
+	}
+}
+
 void Document::NotifyModified(DocModification mh) {
 	if (FlagSet(mh.modificationType, ModificationFlags::InsertText)) {
 		decorations->InsertSpace(mh.position, mh.length);
@@ -3206,8 +3363,8 @@ bool MatchOnLines(const Document *doc, const Regex &regexp, const RESearchRange
 		const Sci::Position lineStartPos = doc->LineStart(line);
 		const Sci::Position lineEndPos = doc->LineEnd(line);
 		const Range lineRange = resr.LineRange(line, lineStartPos, lineEndPos);
-		Iterator itStart(doc, lineRange.start);
-		Iterator itEnd(doc, lineRange.end);
+		const Iterator itStart(doc, lineRange.start);
+		const Iterator itEnd(doc, lineRange.end);
 		const std::regex_constants::match_flag_type flagsMatch = MatchFlags(doc, lineRange.start, lineRange.end, lineStartPos, lineEndPos);
 		std::regex_iterator<Iterator> it(itStart, itEnd, regexp, flagsMatch);
 		for (const std::regex_iterator<Iterator> last; it != last; ++it) {

scintilla/src/Document.h

@@ -172,6 +172,22 @@ public:
 	bool isEnabled;
 };
 
+// Base class for view state that can be held and transferred without understanding the contents.
+// Declared here but real implementation subclass declared in EditModel 
+struct ViewState {
+	ViewState() noexcept = default;
+	// Deleted so ViewState objects can not be copied
+	ViewState(const ViewState &) = delete;
+	ViewState(ViewState &&) = delete;
+	ViewState &operator=(const ViewState &) = delete;
+	ViewState &operator=(ViewState &&) = delete;
+	virtual ~ViewState() noexcept = default;
+
+	virtual void TruncateUndo(int index)=0;
+};
+
+using ViewStateShared = std::shared_ptr<ViewState>;
+
 struct LexerReleaser {
 	// Called by unique_ptr to destroy/free the Resource
 	void operator()(Scintilla::ILexer5 *pLexer) noexcept {
@@ -257,6 +273,8 @@ struct CharacterExtracted {
 	}
 };
 
+bool DiscardLastCombinedCharacter(std::string_view &text) noexcept;
+
 /**
  */
 class Document : PerLine, public Scintilla::IDocument, public Scintilla::ILoader, public Scintilla::IDocumentEditable {
@@ -304,6 +322,8 @@ private:
 	std::unique_ptr<RegexSearchBase> regex;
 	std::unique_ptr<LexInterface> pli;
 
+	std::map<void *, ViewStateShared>viewData;
+
 public:
 
 	Scintilla::EndOfLine eolMode;
@@ -396,8 +416,9 @@ public:
 	}
 	bool IsCollectingUndo() const noexcept { return cb.IsCollectingUndo(); }
 	void BeginUndoAction(bool coalesceWithPrior=false) noexcept { cb.BeginUndoAction(coalesceWithPrior); }
-	void EndUndoAction() noexcept { cb.EndUndoAction(); }
+	void EndUndoAction() noexcept;
 	int UndoSequenceDepth() const noexcept;
+	bool AfterUndoSequenceStart() const noexcept { return cb.AfterUndoSequenceStart(); }
 	void AddUndoAction(Sci::Position token, bool mayCoalesce) { cb.AddUndoAction(token, mayCoalesce); }
 	void SetSavePoint();
 	bool IsSavePoint() const noexcept { return cb.IsSavePoint(); }
@@ -534,6 +555,10 @@ public:
 	LexInterface *GetLexInterface() const noexcept;
 	void SetLexInterface(std::unique_ptr<LexInterface> pLexInterface) noexcept;
 
+	void SetViewState(void *view, ViewStateShared pVSS);
+	ViewStateShared GetViewState(void *view) const noexcept;
+	void TruncateUndoComments(int action);
+
 	int SCI_METHOD SetLineState(Sci_Position line, int state) override;
 	int SCI_METHOD GetLineState(Sci_Position line) const override;
 	Sci::Line GetMaxLineState() const noexcept;
@@ -574,6 +599,7 @@ public:
 private:
 	void NotifyModifyAttempt();
 	void NotifySavePoint(bool atSavePoint);
+	void NotifyGroupCompleted() noexcept;
 	void NotifyModified(DocModification mh);
 };
 
@@ -664,6 +690,7 @@ public:
 	virtual void NotifyDeleted(Document *doc, void *userData) noexcept = 0;
 	virtual void NotifyStyleNeeded(Document *doc, void *userData, Sci::Position endPos) = 0;
 	virtual void NotifyErrorOccurred(Document *doc, void *userData, Scintilla::Status status) = 0;
+	virtual void NotifyGroupCompleted(Document *doc, void *userData) noexcept = 0;
 };
 
 }

scintilla/src/EditModel.cxx

@@ -58,6 +58,42 @@ using namespace Scintilla::Internal;
 Caret::Caret() noexcept :
 	active(false), on(false), period(500) {}
 
+void ModelState::RememberSelectionForUndo(int index, const Selection &sel) {
+	historyForUndo.indexCurrent = index;
+	historyForUndo.ssCurrent = sel.ToString();
+}
+
+void ModelState::ForgetSelectionForUndo() noexcept {
+	historyForUndo.indexCurrent = -1;
+}
+
+void ModelState::RememberSelectionOntoStack(int index, Sci::Line topLine) {
+	if ((historyForUndo.indexCurrent >= 0) && (index == historyForUndo.indexCurrent + 1)) {
+		// Don't overwrite initial selection save if most recent action was coalesced
+		historyForUndo.stack[index] = { historyForUndo.ssCurrent, topLine };
+	}
+}
+
+void ModelState::RememberSelectionForRedoOntoStack(int index, const Selection &sel, Sci::Line topLine) {
+	historyForRedo.stack[index] = { sel.ToString(), topLine };
+}
+
+SelectionWithScroll ModelState::SelectionFromStack(int index, UndoRedo history) const {
+	const SelectionHistory &sh = history == UndoRedo::undo ? historyForUndo : historyForRedo;
+	const SelectionStack::const_iterator it = sh.stack.find(index);
+	if (it != sh.stack.end()) {
+		return it->second;
+	}
+	return {};
+}
+
+void ModelState::TruncateUndo(int index) {
+	const SelectionStack::const_iterator itUndo = historyForUndo.stack.find(index);
+	historyForUndo.stack.erase(itUndo, historyForUndo.stack.end());
+	const SelectionStack::const_iterator itRedo = historyForRedo.stack.find(index);
+	historyForRedo.stack.erase(itRedo, historyForRedo.stack.end());
+}
+
 EditModel::EditModel() : braces{} {
 	inOverstrike = false;
 	xOffset = 0;
@@ -85,6 +121,9 @@ EditModel::EditModel() : braces{} {
 
 EditModel::~EditModel() {
 	try {
+		// Erasing the view state won't throw even though SetViewState
+		// and the resulting map::erase aren't marked noexcept.
+		pdoc->SetViewState(this, {});
 		// This never throws but isn't marked noexcept for compatibility
 		pdoc->Release();
 	} catch (...) {
@@ -131,3 +170,22 @@ InSelection EditModel::LineEndInSelection(Sci::Line lineDoc) const {
 int EditModel::GetMark(Sci::Line line) const {
 	return pdoc->GetMark(line, FlagSet(changeHistoryOption, ChangeHistoryOption::Markers));
 }
+
+void EditModel::EnsureModelState() {
+	if (!modelState && (undoSelectionHistoryOption == UndoSelectionHistoryOption::Enabled)) {
+		if (ViewStateShared vss = pdoc->GetViewState(this)) {
+			modelState = std::dynamic_pointer_cast<ModelState>(vss);
+		} else {
+			modelState = std::make_shared<ModelState>();
+			pdoc->SetViewState(this, std::static_pointer_cast<ViewState>(modelState));
+		}
+	}
+}
+
+void EditModel::ChangeUndoSelectionHistory(Scintilla::UndoSelectionHistoryOption undoSelectionHistoryOptionNew) {
+	undoSelectionHistoryOption = undoSelectionHistoryOptionNew;
+	if (undoSelectionHistoryOption == UndoSelectionHistoryOption::Disabled) {
+		modelState.reset();
+		pdoc->SetViewState(this, {});
+	}
+}

scintilla/src/EditModel.h

@@ -21,6 +21,36 @@ public:
 	Caret() noexcept;
 };
 
+enum class UndoRedo { undo, redo };
+
+// Selection stack is sparse so use a map
+
+struct SelectionWithScroll {
+	std::string selection;
+	Sci::Line topLine = 0;
+};
+
+using SelectionStack = std::map<int, SelectionWithScroll>;
+
+struct SelectionHistory {
+	int indexCurrent = 0;
+	std::string ssCurrent;
+	SelectionStack stack;
+};
+
+struct ModelState : ViewState {
+	SelectionHistory historyForUndo;
+	SelectionHistory historyForRedo;
+	void RememberSelectionForUndo(int index, const Selection &sel);
+	void ForgetSelectionForUndo() noexcept;
+	void RememberSelectionOntoStack(int index, Sci::Line topLine);
+	void RememberSelectionForRedoOntoStack(int index, const Selection &sel, Sci::Line topLine);
+	SelectionWithScroll SelectionFromStack(int index, UndoRedo history) const;
+	virtual void TruncateUndo(int index) final;
+};
+
+using ModelStateShared = std::shared_ptr<ModelState>;
+
 class EditModel {
 public:
 	bool inOverstrike;
@@ -57,6 +87,10 @@ public:
 
 	Document *pdoc;
 
+	Scintilla::UndoSelectionHistoryOption undoSelectionHistoryOption = UndoSelectionHistoryOption::Disabled;
+	bool needRedoRemembered = false;
+	ModelStateShared modelState;
+
 	EditModel();
 	// Deleted so EditModel objects can not be copied.
 	EditModel(const EditModel &) = delete;
@@ -75,6 +109,9 @@ public:
 	const char *GetFoldDisplayText(Sci::Line lineDoc) const noexcept;
 	InSelection LineEndInSelection(Sci::Line lineDoc) const;
 	[[nodiscard]] int GetMark(Sci::Line line) const;
+
+	void EnsureModelState();
+	void ChangeUndoSelectionHistory(Scintilla::UndoSelectionHistoryOption undoSelectionHistoryOptionNew);
 };
 
 }

scintilla/src/EditView.cxx

@@ -2530,19 +2530,13 @@ void EditView::PaintText(Surface *surfaceWindow, const EditModel &model, const V
 
 		Sci::Line lineDocPrevious = -1;	// Used to avoid laying out one document line multiple times
 		std::shared_ptr<LineLayout> ll;
-		std::vector<DrawPhase> phases;
+		DrawPhase phase = DrawPhase::all;
 		if ((phasesDraw == PhasesDraw::Multiple) && !bufferedDraw) {
-			for (DrawPhase phase = DrawPhase::back; phase <= DrawPhase::carets; phase = static_cast<DrawPhase>(static_cast<int>(phase) * 2)) {
-				phases.push_back(phase);
-			}
-		} else {
-			phases.push_back(DrawPhase::all);
+			phase = DrawPhase::back;
 		}
-		for (const DrawPhase &phase : phases) {
-			int ypos = 0;
-			if (!bufferedDraw)
-				ypos += screenLinePaintFirst * vsDraw.lineHeight;
+		for (;;) {
 			int yposScreen = screenLinePaintFirst * vsDraw.lineHeight;
+			int ypos = bufferedDraw ? 0 : yposScreen;
 			Sci::Line visibleLine = model.TopLineOfMain() + screenLinePaintFirst;
 			while (visibleLine < model.pcs->LinesDisplayed() && yposScreen < rcArea.bottom) {
 
@@ -2561,6 +2555,10 @@ void EditView::PaintText(Surface *surfaceWindow, const EditModel &model, const V
 					ll = RetrieveLineLayout(lineDoc, model);
 					LayoutLine(model, surface, vsDraw, ll.get(), model.wrapWidth);
 					lineDocPrevious = lineDoc;
+					if (ll && model.BidirectionalEnabled()) {
+						// Fill the line bidi data
+						UpdateBidiData(model, vsDraw, ll.get());
+					}
 				}
 #if defined(TIME_PAINTING)
 				durLayout += ep.Duration(true);
@@ -2588,11 +2586,6 @@ void EditView::PaintText(Surface *surfaceWindow, const EditModel &model, const V
 						surface->FillRectangleAligned(rcSpacer, Fill(vsDraw.styles[StyleDefault].back));
 					}
 
-					if (model.BidirectionalEnabled()) {
-						// Fill the line bidi data
-						UpdateBidiData(model, vsDraw, ll.get());
-					}
-
 					DrawLine(surface, model, vsDraw, ll.get(), lineDoc, visibleLine, xStart, rcLine, subLine, phase);
 #if defined(TIME_PAINTING)
 					durPaint += ep.Duration(true);
@@ -2631,6 +2624,11 @@ void EditView::PaintText(Surface *surfaceWindow, const EditModel &model, const V
 				yposScreen += vsDraw.lineHeight;
 				visibleLine++;
 			}
+
+			if (phase >= DrawPhase::carets) {
+				break;
+			}
+			phase = static_cast<DrawPhase>(static_cast<int>(phase) * 2);
 		}
 		ll.reset();
 #if defined(TIME_PAINTING)

scintilla/src/Editor.cxx

@@ -711,6 +711,15 @@ void Editor::SetEmptySelection(Sci::Position currentPos_) {
 	SetEmptySelection(SelectionPosition(currentPos_));
 }
 
+void Editor::SetSelectionFromSerialized(const char *serialized) {
+	if (serialized) {
+		sel = Selection(serialized);
+		sel.Truncate(pdoc->Length());
+		SetRectangularRange();
+		InvalidateStyleRedraw();
+	}
+}
+
 void Editor::MultipleSelectAdd(AddNumber addNumber) {
 	if (SelectionEmpty() || !multipleSelection) {
 		// Select word at caret
@@ -926,6 +935,37 @@ void Editor::SetLastXChosen() {
 	lastXChosen = static_cast<int>(pt.x) + xOffset;
 }
 
+void Editor::RememberSelectionForUndo(int index) {
+	EnsureModelState();
+	if (modelState) {
+		modelState->RememberSelectionForUndo(index, sel);
+		needRedoRemembered = true;
+		// Remember selection at end of processing current message
+	}
+}
+
+void Editor::RememberSelectionOntoStack(int index) {
+	EnsureModelState();
+	if (modelState) {
+		// Is undo currently inside a group?
+		if (!pdoc->AfterUndoSequenceStart()) {
+			// Don't remember selections inside a grouped sequence as can only
+			// unto or redo to the start and end of the group.
+			modelState->RememberSelectionOntoStack(index, topLine);
+		}
+	}
+}
+
+void Editor::RememberCurrentSelectionForRedoOntoStack() {
+	if (needRedoRemembered && (pdoc->UndoSequenceDepth() == 0)) {
+		EnsureModelState();
+		if (modelState) {
+			modelState->RememberSelectionForRedoOntoStack(pdoc->UndoCurrent(), sel, topLine);
+			needRedoRemembered = false;
+		}
+	}
+}
+
 void Editor::ScrollTo(Sci::Line line, bool moveThumb) {
 	const Sci::Line topLineNew = std::clamp<Sci::Line>(line, 0, MaxScrollPos());
 	if (topLineNew != topLine) {
@@ -2085,13 +2125,13 @@ void Editor::InsertCharacter(std::string_view sv, CharacterSource charSource) {
 				}
 			}
 		}
+		ThinRectangularRange();
 	}
 	if (wrapOccurred) {
 		SetScrollBars();
 		SetVerticalScrollPos();
 		Redraw();
 	}
-	ThinRectangularRange();
 	// If in wrap mode rewrap current line so EnsureCaretVisible has accurate information
 	EnsureCaretVisible();
 	// Avoid blinking during rapid typing:
@@ -2367,22 +2407,44 @@ void Editor::SelectAll() {
 	Redraw();
 }
 
+void Editor::RestoreSelection(Sci::Position newPos, UndoRedo history) {
+	EnsureModelState();
+	if ((undoSelectionHistoryOption == UndoSelectionHistoryOption::Enabled) && modelState) {
+		// Undo wants the element after the current as it just undid it
+		const int index = pdoc->UndoCurrent() + (history == UndoRedo::undo ? 1 : 0);
+		const SelectionWithScroll selAndLine = modelState->SelectionFromStack(index, history);
+		if (!selAndLine.selection.empty()) {
+			ScrollTo(selAndLine.topLine);
+			sel = Selection(selAndLine.selection);
+			if (sel.IsRectangular()) {
+				const size_t mainForRectangular = sel.Main();
+				// Reconstitute ranges from rectangular range
+				SetRectangularRange();
+				// Restore main if possible.
+				if (mainForRectangular < sel.Count()) {
+					sel.SetMain(mainForRectangular);
+				}
+			}
+			newPos = -1; // Used selection from stack so don't use position returned from undo/redo.
+		}
+	}
+	if (newPos >= 0)
+		SetEmptySelection(newPos);
+	EnsureCaretVisible();
+}
+
 void Editor::Undo() {
 	if (pdoc->CanUndo()) {
 		InvalidateCaret();
 		const Sci::Position newPos = pdoc->Undo();
-		if (newPos >= 0)
-			SetEmptySelection(newPos);
-		EnsureCaretVisible();
+		RestoreSelection(newPos, UndoRedo::undo);
 	}
 }
 
 void Editor::Redo() {
 	if (pdoc->CanRedo()) {
 		const Sci::Position newPos = pdoc->Redo();
-		if (newPos >= 0)
-			SetEmptySelection(newPos);
-		EnsureCaretVisible();
+		RestoreSelection(newPos, UndoRedo::redo);
 	}
 }
 
@@ -2458,6 +2520,17 @@ void Editor::NotifyErrorOccurred(Document *, void *, Status status) {
 	errorStatus = status;
 }
 
+void Editor::NotifyGroupCompleted(Document *, void *) noexcept {
+	// RememberCurrentSelectionForRedoOntoStack may throw (for memory exhaustion)
+	// but this method may not as it is called in UndoGroup destructor so ignore
+	// exception.
+	try {
+		RememberCurrentSelectionForRedoOntoStack();
+	} catch (...) {
+		// Ignore any exception
+	}
+}
+
 void Editor::NotifyChar(int ch, CharacterSource charSource) {
 	NotificationData scn = {};
 	scn.nmhdr.code = Notification::CharAdded;
@@ -2726,6 +2799,15 @@ void Editor::NotifyModified(Document *, DocModification mh, void *) {
 			view.llc.Invalidate(LineLayout::ValidLevel::checkTextAndStyle);
 		}
 	} else {
+		if ((undoSelectionHistoryOption == UndoSelectionHistoryOption::Enabled) &&
+			FlagSet(mh.modificationType, ModificationFlags::User)) {
+			if (FlagSet(mh.modificationType, ModificationFlags::BeforeInsert | ModificationFlags::BeforeDelete)) {
+				RememberSelectionForUndo(pdoc->UndoCurrent());
+			}
+			if (FlagSet(mh.modificationType, ModificationFlags::InsertText | ModificationFlags::DeleteText)) {
+				RememberSelectionOntoStack(pdoc->UndoCurrent());
+			}
+		}
 		// Move selection and brace highlights
 		if (FlagSet(mh.modificationType, ModificationFlags::InsertText)) {
 			sel.MovePositions(true, mh.position, mh.length);
@@ -5460,6 +5542,7 @@ void Editor::SetDocPointer(Document *document) {
 		pdoc = document;
 	}
 	pdoc->AddRef();
+	modelState.reset();
 	pcs = ContractionStateCreate(pdoc->IsLarge());
 
 	// Ensure all positions within document
@@ -8421,8 +8504,8 @@ sptr_t Editor::WndProc(Message iMessage, uptr_t wParam, sptr_t lParam) {
 	case Message::GetLineSelStartPosition:
 	case Message::GetLineSelEndPosition: {
 			const SelectionSegment segmentLine(
-				SelectionPosition(pdoc->LineStart(LineFromUPtr(wParam))),
-				SelectionPosition(pdoc->LineEnd(LineFromUPtr(wParam))));
+				pdoc->LineStart(LineFromUPtr(wParam)),
+				pdoc->LineEnd(LineFromUPtr(wParam)));
 			for (size_t r=0; r<sel.Count(); r++) {
 				const SelectionSegment portion = sel.Range(r).Intersect(segmentLine);
 				if (portion.start.IsValid()) {
@@ -8615,6 +8698,22 @@ sptr_t Editor::WndProc(Message iMessage, uptr_t wParam, sptr_t lParam) {
 	case Message::GetChangeHistory:
 		return static_cast<sptr_t>(changeHistoryOption);
 
+	case Message::SetUndoSelectionHistory:
+		ChangeUndoSelectionHistory(static_cast<UndoSelectionHistoryOption>(wParam));
+		break;
+
+	case Message::GetUndoSelectionHistory:
+		return static_cast<sptr_t>(undoSelectionHistoryOption);
+
+	case Message::SetSelectionSerialized:
+		SetSelectionFromSerialized(ConstCharPtrFromSPtr(lParam));
+		break;
+
+	case Message::GetSelectionSerialized: {
+		const std::string serialized = sel.ToString();
+		return BytesResult(lParam, serialized);
+	}
+
 	case Message::SetExtraAscent:
 		vs.extraAscent = static_cast<int>(wParam);
 		InvalidateStyleRedraw();
@@ -9024,6 +9123,11 @@ sptr_t Editor::WndProc(Message iMessage, uptr_t wParam, sptr_t lParam) {
 	default:
 		return DefWndProc(iMessage, wParam, lParam);
 	}
+
+	// If there was a change that needs its selection saved and it wasn't explicity saved
+	// then do that here.
+	RememberCurrentSelectionForRedoOntoStack();
+
 	//Platform::DebugPrintf("end wnd proc\n");
 	return 0;
 }

scintilla/src/Editor.h

@@ -352,6 +352,7 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	void SetSelection(SelectionPosition currentPos_);
 	void SetEmptySelection(SelectionPosition currentPos_);
 	void SetEmptySelection(Sci::Position currentPos_);
+	void SetSelectionFromSerialized(const char *serialized);
 	enum class AddNumber { one, each };
 	void MultipleSelectAdd(AddNumber addNumber);
 	bool RangeContainsProtected(Sci::Position start, Sci::Position end) const noexcept;
@@ -367,6 +368,9 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	SelectionPosition MovePositionSoVisible(Sci::Position pos, int moveDir);
 	Point PointMainCaret();
 	void SetLastXChosen();
+	void RememberSelectionForUndo(int index);
+	void RememberSelectionOntoStack(int index);
+	void RememberCurrentSelectionForRedoOntoStack();
 
 	void ScrollTo(Sci::Line line, bool moveThumb=true);
 	virtual void ScrollText(Sci::Line linesToMove);
@@ -443,6 +447,7 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	virtual void Paste() = 0;
 	void Clear();
 	virtual void SelectAll();
+	void RestoreSelection(Sci::Position newPos, UndoRedo history);
 	virtual void Undo();
 	virtual void Redo();
 	void DelCharBack(bool allowLineStartDeletion);
@@ -477,6 +482,7 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	void NotifyDeleted(Document *document, void *userData) noexcept override;
 	void NotifyStyleNeeded(Document *doc, void *userData, Sci::Position endStyleNeeded) override;
 	void NotifyErrorOccurred(Document *doc, void *userData, Scintilla::Status status) override;
+	void NotifyGroupCompleted(Document *, void *) noexcept override;
 	void NotifyMacroRecord(Scintilla::Message iMessage, Scintilla::uptr_t wParam, Scintilla::sptr_t lParam);
 
 	void ContainerNeedsUpdate(Scintilla::Update flags) noexcept;

scintilla/src/Platform.h

@@ -194,8 +194,8 @@ public:
 	virtual ~Surface() noexcept = default;
 	static std::unique_ptr<Surface> Allocate(Scintilla::Technology technology);
 
-	virtual void Init(WindowID wid)=0;
-	virtual void Init(SurfaceID sid, WindowID wid)=0;
+	virtual void Init(WindowID wid)=0;	// For measuring text
+	virtual void Init(SurfaceID sid, WindowID wid)=0;	// For drawing
 	virtual std::unique_ptr<Surface> AllocatePixMap(int width, int height)=0;
 
 	virtual void SetMode(SurfaceMode mode)=0;

scintilla/src/PositionCache.cxx

@@ -155,7 +155,7 @@ int LineLayout::LineStart(int line) const noexcept {
 int LineLayout::LineLength(int line) const noexcept {
 	if (!lineStarts) {
 		return numCharsInLine;
-	} if (line >= lines - 1) {
+	} else if (line >= lines - 1) {
 		return numCharsInLine - lineStarts[line];
 	} else {
 		return lineStarts[line + 1] - lineStarts[line];
@@ -356,6 +356,7 @@ void LineLayout::WrapLine(const Document *pdoc, Sci::Position posLineStart, Wrap
 			if (p > 0) {
 				lastGoodBreak = CharacterBoundary(p, -1);
 			}
+			bool foundBreak = false;
 			if (wrapState != Wrap::Char) {
 				Sci::Position pos = lastGoodBreak;
 				while (pos > lastLineStart) {
@@ -370,12 +371,23 @@ void LineLayout::WrapLine(const Document *pdoc, Sci::Position posLineStart, Wrap
 				}
 				if (pos > lastLineStart) {
 					lastGoodBreak = pos;
+					foundBreak = true;
 				}
 			}
-			if (lastGoodBreak == lastLineStart) {
-				// Try moving to start of last character
-				if (p > 0) {
-					lastGoodBreak = CharacterBoundary(p, -1);
+			if (!foundBreak) {
+				if (CpUtf8 == pdoc->dbcsCodePage) {
+					// Go back before a base character, commonly a letter as modifiers are after the letter they modify
+					std::string_view svWithoutLast(&chars[lastLineStart], CharacterBoundary(p + 1, 1) - lastLineStart);
+					if (DiscardLastCombinedCharacter(svWithoutLast) && !svWithoutLast.empty()) {
+						lastGoodBreak = lastLineStart + static_cast<Sci::Position>(svWithoutLast.length());
+						foundBreak = true;
+					}
+				}
+				if (!foundBreak) {
+					// Try moving to start of last character
+					if (p > 0) {
+						lastGoodBreak = CharacterBoundary(p, -1);
+					}
 				}
 				if (lastGoodBreak == lastLineStart) {
 					// Ensure at least one character on line.
@@ -409,8 +421,7 @@ ScreenLine::ScreenLine(
 	tabWidthMinimumPixels(tabWidthMinimumPixels_) {
 }
 
-ScreenLine::~ScreenLine() {
-}
+ScreenLine::~ScreenLine() = default;
 
 std::string_view ScreenLine::Text() const {
 	return std::string_view(&ll->chars[start], len);
@@ -498,7 +509,6 @@ bool AllGraphicASCII(std::string_view text) {
 size_t LineLayoutCache::EntryForLine(Sci::Line line) const noexcept {
 	switch (level) {
 	case LineCache::None:
-		return 0;
 	case LineCache::Caret:
 		return 0;
 	case LineCache::Page:
@@ -652,9 +662,10 @@ namespace {
 // Simply pack the (maximum 4) character bytes into an int
 constexpr unsigned int KeyFromString(std::string_view charBytes) noexcept {
 	PLATFORM_ASSERT(charBytes.length() <= 4);
+	constexpr int byteMultiplier = 0x100;
 	unsigned int k=0;
 	for (const unsigned char uc : charBytes) {
-		k = k * 0x100 + uc;
+		k = k * byteMultiplier + uc;
 	}
 	return k;
 }
@@ -682,16 +693,16 @@ namespace Scintilla::Internal {
 const char *ControlCharacterString(unsigned char ch) noexcept {
 	if (ch < std::size(repsC0)) {
 		return repsC0[ch];
-	} else {
-		return "BAD";
 	}
+	return "BAD";
 }
 
 
 void Hexits(char *hexits, int ch) noexcept {
+	constexpr int hexDivisor = 0x10;
 	hexits[0] = 'x';
-	hexits[1] = "0123456789ABCDEF"[ch / 0x10];
-	hexits[2] = "0123456789ABCDEF"[ch % 0x10];
+	hexits[1] = "0123456789ABCDEF"[ch / hexDivisor];
+	hexits[2] = "0123456789ABCDEF"[ch % hexDivisor];
 	hexits[3] = 0;
 }
 
@@ -856,16 +867,12 @@ BreakFinder::BreakFinder(const LineLayout *ll_, const Selection *psel, Range lin
 	}
 
 	if (FlagSet(breakFor, BreakFor::Selection)) {
-		const SelectionPosition posStart(posLineStart);
-		const SelectionPosition posEnd(posLineStart + lineRange.end);
-		const SelectionSegment segmentLine(posStart, posEnd);
+		const SelectionSegment segmentLine(posLineStart, posLineStart + lineRange.end);
 		for (size_t r=0; r<psel->Count(); r++) {
 			const SelectionSegment portion = psel->Range(r).Intersect(segmentLine);
-			if (!(portion.start == portion.end)) {
-				if (portion.start.IsValid())
-					Insert(portion.start.Position() - posLineStart);
-				if (portion.end.IsValid())
-					Insert(portion.end.Position() - posLineStart);
+			if (!portion.Empty()) {
+				Insert(portion.start.Position() - posLineStart);
+				Insert(portion.end.Position() - posLineStart);
 			}
 		}
 		// On the curses platform, the terminal is drawing its own caret, so add breaks around the
@@ -988,10 +995,10 @@ bool BreakFinder::More() const noexcept {
 }
 
 class PositionCacheEntry {
-	uint16_t styleNumber;
-	uint16_t len;
-	uint16_t clock;
-	bool unicode;
+	uint16_t styleNumber = 0;
+	uint16_t len = 0;
+	uint16_t clock = 0;
+	bool unicode = false;
 	std::unique_ptr<XYPOSITION[]> positions;
 public:
 	PositionCacheEntry() noexcept;
@@ -1006,15 +1013,16 @@ public:
 	void Clear() noexcept;
 	bool Retrieve(unsigned int styleNumber_, bool unicode_, std::string_view sv, XYPOSITION *positions_) const noexcept;
 	static size_t Hash(unsigned int styleNumber_, bool unicode_, std::string_view sv) noexcept;
-	bool NewerThan(const PositionCacheEntry &other) const noexcept;
+	[[nodiscard]] bool NewerThan(const PositionCacheEntry &other) const noexcept;
 	void ResetClock() noexcept;
 };
 
 class PositionCache : public IPositionCache {
-	std::vector<PositionCacheEntry> pces;
+	static constexpr size_t defaultCacheSize = 0x400;
+	std::vector<PositionCacheEntry> pces{ defaultCacheSize };
 	std::mutex mutex;
-	uint16_t clock;
-	bool allClear;
+	uint16_t clock = 1;
+	bool allClear = true;
 public:
 	PositionCache();
 	// Deleted so LineAnnotation objects can not be copied.
@@ -1026,14 +1034,12 @@ public:
 
 	void Clear() noexcept override;
 	void SetSize(size_t size_) override;
-	size_t GetSize() const noexcept override;
+	[[nodiscard]] size_t GetSize() const noexcept override;
 	void MeasureWidths(Surface *surface, const ViewStyle &vstyle, unsigned int styleNumber,
 		bool unicode, std::string_view sv, XYPOSITION *positions, bool needsLocking) override;
 };
 
-PositionCacheEntry::PositionCacheEntry() noexcept :
-	styleNumber(0), len(0), clock(0), unicode(false) {
-}
+PositionCacheEntry::PositionCacheEntry() noexcept = default;
 
 // Copy constructor not currently used, but needed for being element in std::vector.
 PositionCacheEntry::PositionCacheEntry(const PositionCacheEntry &other) :
@@ -1079,9 +1085,8 @@ bool PositionCacheEntry::Retrieve(unsigned int styleNumber_, bool unicode_, std:
 			positions_[i] = positions[i];
 		}
 		return true;
-	} else {
-		return false;
 	}
+	return false;
 }
 
 size_t PositionCacheEntry::Hash(unsigned int styleNumber_, bool unicode_, std::string_view sv) noexcept {
@@ -1100,11 +1105,7 @@ void PositionCacheEntry::ResetClock() noexcept {
 	}
 }
 
-PositionCache::PositionCache() {
-	clock = 1;
-	pces.resize(0x400);
-	allClear = true;
-}
+PositionCache::PositionCache() = default;
 
 void PositionCache::Clear() noexcept {
 	if (!allClear) {

scintilla/src/Selection.cxx

@@ -9,11 +9,13 @@
 #include <cstdlib>
 
 #include <stdexcept>
+#include <string>
 #include <string_view>
 #include <vector>
 #include <optional>
 #include <algorithm>
 #include <memory>
+#include <charconv>
 
 #include "Debugging.h"
 
@@ -22,6 +24,30 @@
 
 using namespace Scintilla::Internal;
 
+namespace {
+
+// Generically convert a string to a integer value throwing if the conversion failed.
+// Failures include values that are out of range for the destination variable.
+template <typename T>
+void ValueFromString(std::string_view sv, T &value) {
+	const std::from_chars_result res = std::from_chars(sv.data(), sv.data() + sv.size(), value);
+	if (res.ec != std::errc{}) {
+		if (res.ec == std::errc::result_out_of_range)
+			throw std::runtime_error("from_chars out of range.");
+		throw std::runtime_error("from_chars failed.");
+	}
+}
+
+}
+
+SelectionPosition::SelectionPosition(std::string_view sv) : position(0) {
+	if (const size_t v = sv.find('v'); v != std::string_view::npos) {
+		ValueFromString(sv.substr(v + 1), virtualSpace);
+		sv = sv.substr(0, v);
+	}
+	ValueFromString(sv, position);
+}
+
 void SelectionPosition::MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length, bool moveForEqual) noexcept {
 	if (insertion) {
 		if (position == startChange) {
@@ -52,32 +78,42 @@ void SelectionPosition::MoveForInsertDelete(bool insertion, Sci::Position startC
 	}
 }
 
-bool SelectionPosition::operator <(const SelectionPosition &other) const noexcept {
-	if (position == other.position)
-		return virtualSpace < other.virtualSpace;
-	else
-		return position < other.position;
-}
-
 bool SelectionPosition::operator >(const SelectionPosition &other) const noexcept {
 	if (position == other.position)
 		return virtualSpace > other.virtualSpace;
-	else
-		return position > other.position;
+	return position > other.position;
 }
 
 bool SelectionPosition::operator <=(const SelectionPosition &other) const noexcept {
-	if (position == other.position && virtualSpace == other.virtualSpace)
+	if (other == *this)
 		return true;
-	else
-		return other > *this;
+	return other > *this;
 }
 
 bool SelectionPosition::operator >=(const SelectionPosition &other) const noexcept {
-	if (position == other.position && virtualSpace == other.virtualSpace)
+	if (other == *this)
 		return true;
-	else
-		return *this > other;
+	return *this > other;
+}
+
+std::string SelectionPosition::ToString() const {
+	std::string result = std::to_string(position);
+	if (virtualSpace) {
+		result += 'v';
+		result += std::to_string(virtualSpace);
+	}
+	return result;
+}
+
+SelectionRange::SelectionRange(std::string_view sv) {
+	const size_t dash = sv.find('-');
+	if (dash == std::string_view::npos) {
+		anchor = SelectionPosition(sv);
+		caret = anchor;
+	} else {
+		anchor = SelectionPosition(sv.substr(0, dash));
+		caret = SelectionPosition(sv.substr(dash + 1));
+	}
 }
 
 Sci::Position SelectionRange::Length() const noexcept {
@@ -131,20 +167,15 @@ bool SelectionRange::ContainsCharacter(SelectionPosition spCharacter) const noex
 }
 
 SelectionSegment SelectionRange::Intersect(SelectionSegment check) const noexcept {
-	const SelectionSegment inOrder(caret, anchor);
-	if ((inOrder.start <= check.end) || (inOrder.end >= check.start)) {
-		SelectionSegment portion = check;
-		if (portion.start < inOrder.start)
-			portion.start = inOrder.start;
-		if (portion.end > inOrder.end)
-			portion.end = inOrder.end;
-		if (portion.start > portion.end)
-			return SelectionSegment();
-		else
-			return portion;
-	} else {
-		return SelectionSegment();
+	const SelectionSegment inOrder = AsSegment();
+	if ((inOrder.start > check.end) || (inOrder.end < check.start)) {
+		// Nothing in common, not even touching so return empty *invalid* segment
+		return {};
 	}
+	return {
+		std::max(check.start, inOrder.start),
+		std::min(check.end, inOrder.end)
+	};
 }
 
 void SelectionRange::Swap() noexcept {
@@ -181,9 +212,15 @@ bool SelectionRange::Trim(SelectionRange range) noexcept {
 			caret = end;
 		}
 		return Empty();
-	} else {
-		return false;
 	}
+	return false;
+}
+
+void SelectionRange::Truncate(Sci::Position length) noexcept {
+	if (anchor.Position() > length)
+		anchor.SetPosition(length);
+	if (caret.Position() > length)
+		caret.SetPosition(length);
 }
 
 // If range is all virtual collapse to start of virtual space
@@ -197,10 +234,73 @@ void SelectionRange::MinimizeVirtualSpace() noexcept {
 	}
 }
 
+std::string SelectionRange::ToString() const {
+	std::string result = anchor.ToString();
+	if (!(caret == anchor)) {
+		result += '-';
+		result += caret.ToString();
+	}
+	return result;
+}
+
 Selection::Selection() : mainRange(0), moveExtends(false), tentativeMain(false), selType(SelTypes::stream) {
 	AddSelection(SelectionRange(SelectionPosition(0)));
 }
 
+Selection::Selection(std::string_view sv) : mainRange(0), moveExtends(false), tentativeMain(false), selType(SelTypes::stream) {
+	if (sv.empty()) {
+		return;
+	}
+	try {
+		// Decode initial letter prefix if any
+		switch (sv.front()) {
+		case 'R':
+			selType = SelTypes::rectangle;
+			break;
+		case 'L':
+			selType = SelTypes::lines;
+			break;
+		case 'T':
+			selType = SelTypes::thin;
+			break;
+		default:
+			break;
+		}
+		if (selType != SelTypes::stream) {
+			sv.remove_prefix(1);
+		}
+
+		// Non-zero main index at end after '#'
+		if (const size_t hash = sv.find('#'); hash != std::string_view::npos) {
+			ValueFromString(sv.substr(hash + 1), mainRange);
+			sv = sv.substr(0, hash);
+		}
+
+		// Remainder is list of ranges
+		if (selType == SelTypes::rectangle || selType == SelTypes::thin) {
+			rangeRectangular = SelectionRange(sv);
+			// Ensure enough ranges exist for mainRange to be in bounds
+			for (size_t i = 0; i <= mainRange; i++) {
+				ranges.emplace_back(SelectionPosition(0));
+			}
+		} else {
+			size_t comma = sv.find(',');
+			while (comma != std::string_view::npos) {
+				ranges.emplace_back(sv.substr(0, comma));
+				sv.remove_prefix(comma + 1);
+				comma = sv.find(',');
+			}
+			ranges.emplace_back(sv);
+			if (mainRange >= ranges.size()) {
+				mainRange = ranges.size() - 1;
+			}
+		}
+	} catch (std::runtime_error &) {
+		// On failure, produce an empty selection.
+		Clear();
+	}
+}
+
 bool Selection::IsRectangular() const noexcept {
 	return (selType == SelTypes::rectangle) || (selType == SelTypes::thin);
 }
@@ -217,9 +317,13 @@ SelectionRange &Selection::Rectangular() noexcept {
 	return rangeRectangular;
 }
 
+SelectionRange Selection::RectangularCopy() const noexcept {
+	return rangeRectangular;
+}
+
 SelectionSegment Selection::Limits() const noexcept {
 	PLATFORM_ASSERT(!ranges.empty());
-	SelectionSegment sr(ranges[0].anchor, ranges[0].caret);
+	SelectionSegment sr = ranges[0].AsSegment();
 	for (size_t i=1; i<ranges.size(); i++) {
 		sr.Extend(ranges[i].anchor);
 		sr.Extend(ranges[i].caret);
@@ -230,9 +334,8 @@ SelectionSegment Selection::Limits() const noexcept {
 SelectionSegment Selection::LimitsForRectangularElseMain() const noexcept {
 	if (IsRectangular()) {
 		return Limits();
-	} else {
-		return SelectionSegment(ranges[mainRange].caret, ranges[mainRange].anchor);
 	}
+	return ranges[mainRange].AsSegment();
 }
 
 size_t Selection::Count() const noexcept {
@@ -267,9 +370,8 @@ const SelectionRange &Selection::RangeMain() const noexcept {
 SelectionPosition Selection::Start() const noexcept {
 	if (IsRectangular()) {
 		return rangeRectangular.Start();
-	} else {
-		return ranges[mainRange].Start();
 	}
+	return ranges[mainRange].Start();
 }
 
 bool Selection::MoveExtends() const noexcept {
@@ -428,11 +530,13 @@ void Selection::Clear() noexcept {
 	if (ranges.size() > 1) {
 		ranges.erase(ranges.begin() + 1, ranges.end());
 	}
-	mainRange = 0;
-	selType = SelTypes::stream;
-	moveExtends = false;
-	ranges[mainRange].Reset();
+	ranges[0].Reset();
+	rangesSaved.clear();
 	rangeRectangular.Reset();
+	mainRange = 0;
+	moveExtends = false;
+	tentativeMain = false;
+	selType = SelTypes::stream;
 }
 
 void Selection::RemoveDuplicates() noexcept {
@@ -456,3 +560,52 @@ void Selection::RotateMain() noexcept {
 	mainRange = (mainRange + 1) % ranges.size();
 }
 
+void Selection::SetRanges(const Ranges &rangesToSet) {
+	ranges = rangesToSet;
+}
+
+void Selection::Truncate(Sci::Position length) noexcept {
+	// This may be needed when applying a persisted selection onto a document that has been shortened.
+	for (SelectionRange &range : ranges) {
+		range.Truncate(length);
+	}
+	// Above may have made some non-unique empty ranges.
+	RemoveDuplicates();
+	rangeRectangular.Truncate(length);
+}
+
+std::string Selection::ToString() const {
+	std::string result;
+	switch (selType) {
+	case SelTypes::rectangle:
+		result += 'R';
+		break;
+	case SelTypes::lines:
+		result += 'L';
+		break;
+	case SelTypes::thin:
+		result += 'T';
+		break;
+	default:
+		// No handling of none as not a real value of enumeration, just used for empty arguments
+		// No prefix.
+		break;
+	}
+	if (selType == SelTypes::rectangle || selType == SelTypes::thin) {
+		result += rangeRectangular.ToString();
+	} else {
+		for (size_t r = 0; r < ranges.size(); r++) {
+			if (r > 0) {
+				result += ',';
+			}
+			result += ranges[r].ToString();
+		}
+	}
+
+	if (mainRange > 0) {
+		result += '#';
+		result += std::to_string(mainRange);
+	}
+
+	return result;
+}

scintilla/src/Selection.h

@@ -11,26 +11,35 @@
 namespace Scintilla::Internal {
 
 class SelectionPosition {
-	Sci::Position position;
-	Sci::Position virtualSpace;
+	Sci::Position position = Sci::invalidPosition;
+	Sci::Position virtualSpace = 0;
 public:
-	explicit SelectionPosition(Sci::Position position_= Sci::invalidPosition, Sci::Position virtualSpace_=0) noexcept : position(position_), virtualSpace(virtualSpace_) {
+	constexpr SelectionPosition() noexcept = default;
+	constexpr explicit SelectionPosition(Sci::Position position_, Sci::Position virtualSpace_=0) noexcept : position(position_), virtualSpace(virtualSpace_) {
 		PLATFORM_ASSERT(virtualSpace < 800000000);
 		if (virtualSpace < 0)
 			virtualSpace = 0;
 	}
+	explicit SelectionPosition(std::string_view sv);
 	void Reset() noexcept {
 		position = 0;
 		virtualSpace = 0;
 	}
 	void MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length, bool moveForEqual) noexcept;
-	bool operator ==(const SelectionPosition &other) const noexcept {
-		return position == other.position && virtualSpace == other.virtualSpace;
+	[[nodiscard]] constexpr bool operator ==(const SelectionPosition &other) const noexcept {
+		return (position == other.position) && (virtualSpace == other.virtualSpace);
 	}
-	bool operator <(const SelectionPosition &other) const noexcept;
-	bool operator >(const SelectionPosition &other) const noexcept;
-	bool operator <=(const SelectionPosition &other) const noexcept;
-	bool operator >=(const SelectionPosition &other) const noexcept;
+	[[nodiscard]] constexpr bool operator !=(const SelectionPosition &other) const noexcept {
+		return (position != other.position) || (virtualSpace != other.virtualSpace);
+	}
+	[[nodiscard]] constexpr bool operator <(const SelectionPosition &other) const noexcept {
+		if (position == other.position)
+			return virtualSpace < other.virtualSpace;
+		return position < other.position;
+	}
+	[[nodiscard]] bool operator >(const SelectionPosition &other) const noexcept;
+	[[nodiscard]] bool operator <=(const SelectionPosition &other) const noexcept;
+	[[nodiscard]] bool operator >=(const SelectionPosition &other) const noexcept;
 	Sci::Position Position() const noexcept {
 		return position;
 	}
@@ -55,15 +64,15 @@ public:
 	bool IsValid() const noexcept {
 		return position >= 0;
 	}
+	std::string ToString() const;
 };
 
 // Ordered range to make drawing simpler
 struct SelectionSegment {
 	SelectionPosition start;
 	SelectionPosition end;
-	SelectionSegment() noexcept : start(), end() {
-	}
-	SelectionSegment(SelectionPosition a, SelectionPosition b) noexcept {
+	constexpr SelectionSegment() noexcept = default;
+	constexpr SelectionSegment(SelectionPosition a, SelectionPosition b) noexcept {
 		if (a < b) {
 			start = a;
 			end = b;
@@ -72,6 +81,12 @@ struct SelectionSegment {
 			end = a;
 		}
 	}
+	constexpr SelectionSegment(Sci::Position a, Sci::Position b) :
+		SelectionSegment(SelectionPosition(a), SelectionPosition(b)) {
+	}
+	[[nodiscard]] constexpr bool operator ==(const SelectionSegment &other) const noexcept {
+		return (start == other.start) && (end == other.end);
+	}
 	bool Empty() const noexcept {
 		return start == end;
 	}
@@ -96,15 +111,18 @@ struct SelectionRange {
 	SelectionPosition caret;
 	SelectionPosition anchor;
 
-	SelectionRange() noexcept : caret(), anchor() {
+	constexpr SelectionRange() noexcept = default;
+	constexpr explicit SelectionRange(SelectionPosition single) noexcept : caret(single), anchor(single) {
 	}
-	explicit SelectionRange(SelectionPosition single) noexcept : caret(single), anchor(single) {
+	constexpr explicit SelectionRange(Sci::Position single) noexcept : caret(single), anchor(single) {
 	}
-	explicit SelectionRange(Sci::Position single) noexcept : caret(single), anchor(single) {
+	constexpr SelectionRange(SelectionPosition caret_, SelectionPosition anchor_) noexcept : caret(caret_), anchor(anchor_) {
 	}
-	SelectionRange(SelectionPosition caret_, SelectionPosition anchor_) noexcept : caret(caret_), anchor(anchor_) {
+	constexpr SelectionRange(Sci::Position caret_, Sci::Position anchor_) noexcept : caret(caret_), anchor(anchor_) {
 	}
-	SelectionRange(Sci::Position caret_, Sci::Position anchor_) noexcept : caret(caret_), anchor(anchor_) {
+	explicit SelectionRange(std::string_view sv);
+	SelectionSegment AsSegment() const noexcept {
+		return {caret, anchor};
 	}
 	bool Empty() const noexcept {
 		return anchor == caret;
@@ -139,16 +157,19 @@ struct SelectionRange {
 	}
 	void Swap() noexcept;
 	bool Trim(SelectionRange range) noexcept;
+	void Truncate(Sci::Position length) noexcept;
 	// If range is all virtual collapse to start of virtual space
 	void MinimizeVirtualSpace() noexcept;
+	std::string ToString() const;
 };
 
 // Deliberately an enum rather than an enum class to allow treating as bool
 enum InSelection { inNone, inMain, inAdditional };
 
 class Selection {
-	std::vector<SelectionRange> ranges;
-	std::vector<SelectionRange> rangesSaved;
+	using Ranges = std::vector<SelectionRange>;
+	Ranges ranges;
+	Ranges rangesSaved;
 	SelectionRange rangeRectangular;
 	size_t mainRange;
 	bool moveExtends;
@@ -157,11 +178,14 @@ public:
 	enum class SelTypes { none, stream, rectangle, lines, thin };
 	SelTypes selType;
 
-	Selection();
+	Selection();	// Allocates so may throw.
+	explicit Selection(std::string_view sv);
+
 	bool IsRectangular() const noexcept;
 	Sci::Position MainCaret() const noexcept;
 	Sci::Position MainAnchor() const noexcept;
 	SelectionRange &Rectangular() noexcept;
+	SelectionRange RectangularCopy() const noexcept;
 	SelectionSegment Limits() const noexcept;
 	// This is for when you want to move the caret in response to a
 	// user direction command - for rectangular selections, use the range
@@ -198,9 +222,12 @@ public:
 	void RemoveDuplicates() noexcept;
 	void RotateMain() noexcept;
 	bool Tentative() const noexcept { return tentativeMain; }
-	std::vector<SelectionRange> RangesCopy() const {
+	Ranges RangesCopy() const {
 		return ranges;
 	}
+	void SetRanges(const Ranges &rangesToSet);
+	void Truncate(Sci::Position length) noexcept;
+	std::string ToString() const;
 };
 
 }

scintilla/src/SplitVector.h

@@ -191,7 +191,7 @@ public:
 			}
 			RoomFor(insertLength);
 			GapTo(position);
-			std::fill(body.data() + part1Length, body.data() + part1Length + insertLength, v);
+			std::fill_n(body.data() + part1Length, insertLength, v);
 			lengthBody += insertLength;
 			part1Length += insertLength;
 			gapLength -= insertLength;
@@ -210,9 +210,10 @@ public:
 			}
 			RoomFor(insertLength);
 			GapTo(position);
-			for (ptrdiff_t elem = part1Length; elem < part1Length + insertLength; elem++) {
+			T *ptr = body.data() + part1Length;
+			for (ptrdiff_t elem = 0; elem < insertLength; elem++, ptr++) {
 				T emptyOne = {};
-				body[elem] = std::move(emptyOne);
+				*ptr = std::move(emptyOne);
 			}
 			lengthBody += insertLength;
 			part1Length += insertLength;
@@ -238,7 +239,7 @@ public:
 			}
 			RoomFor(insertLength);
 			GapTo(positionToInsert);
-			std::copy(s + positionFrom, s + positionFrom + insertLength, body.data() + part1Length);
+			std::copy_n(s + positionFrom, insertLength, body.data() + part1Length);
 			lengthBody += insertLength;
 			part1Length += insertLength;
 			gapLength -= insertLength;
@@ -284,11 +285,11 @@ public:
 			if (range1Length > part1AfterPosition)
 				range1Length = part1AfterPosition;
 		}
-		std::copy(body.data() + position, body.data() + position + range1Length, buffer);
+		std::copy_n(body.data() + position, range1Length, buffer);
 		buffer += range1Length;
 		position = position + range1Length + gapLength;
 		const ptrdiff_t range2Length = retrieveLength - range1Length;
-		std::copy(body.data() + position, body.data() + position + range2Length, buffer);
+		std::copy_n(body.data() + position, range2Length, buffer);
 	}
 
 	/// Compact the buffer and return a pointer to the first element.

scintilla/src/UndoHistory.cxx

@@ -354,6 +354,14 @@ int UndoHistory::UndoSequenceDepth() const noexcept {
 	return undoSequenceDepth;
 }
 
+bool UndoHistory::AfterUndoSequenceStart() const noexcept {
+	if (currentAction == 0) {
+		return false;
+	}
+	// Count back to last sequence start?
+	return !actions.AtStart(currentAction-1);
+}
+
 void UndoHistory::DropUndoSequence() noexcept {
 	undoSequenceDepth = 0;
 }

scintilla/src/UndoHistory.h

@@ -102,6 +102,7 @@ public:
 	void BeginUndoAction(bool mayCoalesce=false) noexcept;
 	void EndUndoAction() noexcept;
 	int UndoSequenceDepth() const noexcept;
+	bool AfterUndoSequenceStart() const noexcept;
 	void DropUndoSequence() noexcept;
 	void DeleteUndoHistory() noexcept;
 

scintilla/src/UniConversion.cxx

@@ -261,6 +261,18 @@ unsigned int UTF16FromUTF32Character(unsigned int val, wchar_t *tbuf) noexcept {
 	return 2;
 }
 
+int UnicodeFromUTF8(std::string_view sv) noexcept {
+	if (!sv.empty()) {
+		const unsigned char uch = sv.front();
+		const unsigned int byteCount = UTF8BytesOfLead[uch];
+		if (sv.length() >= byteCount) {
+			return UnicodeFromUTF8(reinterpret_cast<const unsigned char *>(sv.data()));
+		}
+	}
+	// Failure so let the caller know  
+	return unicodeReplacementChar;
+}
+
 const unsigned char UTF8BytesOfLead[256] = {
 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 00 - 0F
 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 10 - 1F

scintilla/src/UniConversion.h

@@ -43,11 +43,16 @@ inline int UnicodeFromUTF8(const unsigned char *us) noexcept {
 		return ((us[0] & 0x7) << 18) + ((us[1] & 0x3F) << 12) + ((us[2] & 0x3F) << 6) + (us[3] & 0x3F);
 	}
 }
+int UnicodeFromUTF8(std::string_view sv) noexcept;
 
 constexpr bool UTF8IsTrailByte(unsigned char ch) noexcept {
 	return (ch >= 0x80) && (ch < 0xc0);
 }
 
+constexpr bool UTF8IsFirstByte(unsigned char ch) noexcept {
+	return (ch >= 0xc2) && (ch <= 0xf4);
+}
+
 constexpr bool UTF8IsAscii(unsigned char ch) noexcept {
 	return ch < 0x80;
 }

scintilla/test/simpleTests.py

@@ -2242,6 +2242,39 @@ class TestMultiSelection(unittest.TestCase):
 		self.assertEqual(self.ed.Contents(), b'a    1')
 		self.assertEqual(self.textOfSelection(0), b' ')
 
+	def testSelectionSerialization(self):
+		self.ed.SetContents(b"a")
+		self.ed.SetSelection(0, 1)
+		self.assertEqual(self.ed.GetSelectionSerialized(), b'1-0')
+		self.ed.SetSelection(1, 1)
+		self.assertEqual(self.ed.GetSelectionSerialized(), b'1')
+		self.ed.SetSelectionNAnchorVirtualSpace(0, 2)
+		self.ed.SetSelectionNCaretVirtualSpace(0, 3)
+		self.assertEqual(selectionRepresentation(self.ed, 0), "1+2v-1+3v")
+		self.assertEqual(self.textOfSelection(0), b'')
+		self.assertEqual(self.ed.GetSelectionSerialized(), b'1v2-1v3')
+		self.ed.SetSelectionSerialized(0, b'1-0')
+		self.assertEqual(self.ed.MainSelection, 0)
+		self.assertEqual(self.ed.Anchor, 1)
+		self.assertEqual(self.ed.CurrentPos, 0)
+		self.assertEqual(self.ed.GetSelectionNAnchor(0), 1)
+		self.assertEqual(self.ed.GetSelectionNCaret(0), 0)
+
+	def testSelectionSerializationOutOfBounds(self):
+		# Try setting selections that extend past document end through serialized form
+		# and check that the selection is limited to the document.
+		self.ed.SetContents(b"a")
+		self.ed.SetSelectionSerialized(0, b'200-0')
+		self.assertEqual(self.ed.GetSelectionSerialized(), b'1-0')
+		
+		# Retain virtual space
+		self.ed.SetSelectionSerialized(0, b'0v1-200')
+		self.assertEqual(self.ed.GetSelectionSerialized(), b'0v1-1')
+
+		# Drop identical ranges, but touching empty range survives
+		self.ed.SetSelectionSerialized(0, b'0-200,300-400,500-600')
+		self.assertEqual(self.ed.GetSelectionSerialized(), b'0-1,1')
+
 
 class TestModalSelection(unittest.TestCase):
 

scintilla/test/unit/UnitTester.vcxproj

@@ -75,13 +75,14 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
     <EnableClangTidyCodeAnalysis>true</EnableClangTidyCodeAnalysis>
-    <CodeAnalysisRuleSet>..\..\..\..\..\Users\Neil\SensibleRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet>..\..\..\lexilla\src\Lexilla.ruleset</CodeAnalysisRuleSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
+    <CodeAnalysisRuleSet>..\..\..\lexilla\src\Lexilla.ruleset</CodeAnalysisRuleSet>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
@@ -165,6 +166,7 @@
     <ClCompile Include="..\..\src\PerLine.cxx" />
     <ClCompile Include="..\..\src\RESearch.cxx" />
     <ClCompile Include="..\..\src\RunStyles.cxx" />
+    <ClCompile Include="..\..\src\Selection.cxx" />
     <ClCompile Include="..\..\src\UndoHistory.cxx" />
     <ClCompile Include="..\..\src\UniConversion.cxx" />
     <ClCompile Include="..\..\src\UniqueString.cxx" />

scintilla/test/unit/makefile

@@ -68,6 +68,7 @@ Geometry.o \
 PerLine.o \
 RESearch.o \
 RunStyles.o \
+Selection.o \
 UndoHistory.o \
 UniConversion.o \
 UniqueString.o

scintilla/test/unit/test.mak

@@ -25,6 +25,7 @@ TESTEDSRC=\
  ../../src/PerLine.cxx \
  ../../src/RESearch.cxx \
  ../../src/RunStyles.cxx \
+ ../../src/Selection.cxx \
  ../../src/UndoHistory.cxx \
  ../../src/UniConversion.cxx \
  ../../src/UniqueString.cxx

scintilla/test/unit/testCharClassify.cxx

@@ -21,11 +21,16 @@ using namespace Scintilla::Internal;
 
 // Test CharClassify.
 
+constexpr int byteValues = 256;
+
 class CharClassifyTest {
+public:
+	// Avoid warnings, deleted so never called.
+	CharClassifyTest(const CharClassifyTest &) = delete;
 protected:
 	CharClassifyTest() {
 		pcc = std::make_unique<CharClassify>();
-		for (int ch = 0; ch < 256; ch++) {
+		for (int ch = 0; ch < byteValues; ch++) {
 			if (ch == '\r' || ch == '\n')
 				charClass[ch] = CharacterClass::newLine;
 			else if (ch < 0x20 || ch == ' ' || ch == '\x7f')
@@ -36,11 +41,9 @@ protected:
 				charClass[ch] = CharacterClass::punctuation;
 		}
 	}
-	// Avoid warnings, deleted so never called.
-	CharClassifyTest(const CharClassifyTest &) = delete;
 
 	std::unique_ptr<CharClassify> pcc;
-	CharacterClass charClass[256] {};
+	CharacterClass charClass[byteValues] {};
 
 	static const char* GetClassName(CharacterClass charClass) noexcept {
 		switch(charClass) {
@@ -57,37 +60,39 @@ protected:
 };
 
 TEST_CASE_METHOD(CharClassifyTest, "Defaults") {
-	for (int i = 0; i < 256; i++) {
-		if (charClass[i] != pcc->GetClass(i))
+	for (int i = 0; i < byteValues; i++) {
+		if (charClass[i] != pcc->GetClass(i)) {
 			std::cerr
 			<< "Character " << i
 			<< " should be class " << GetClassName(charClass[i])
 			<< ", but got " << GetClassName(pcc->GetClass(i)) << std::endl;
+		}
 		REQUIRE(charClass[i] == pcc->GetClass(i));
 	}
 }
 
 TEST_CASE_METHOD(CharClassifyTest, "Custom") {
 	unsigned char buf[2] = {0, 0};
-	for (int i = 0; i < 256; i++) {
+	for (int i = 0; i < byteValues; i++) {
 		const CharacterClass thisClass = static_cast<CharacterClass>(i % 4);
 		buf[0] = i;
 		pcc->SetCharClasses(buf, thisClass);
 		charClass[i] = thisClass;
 	}
-	for (int i = 0; i < 256; i++) {
-		if (charClass[i] != pcc->GetClass(i))
+	for (int i = 0; i < byteValues; i++) {
+		if (charClass[i] != pcc->GetClass(i)) {
 			std::cerr
 			<< "Character " << i
 			<< " should be class " << GetClassName(charClass[i])
 			<< ", but got " << GetClassName(pcc->GetClass(i)) << std::endl;
+		}
 		REQUIRE(charClass[i] == pcc->GetClass(i));
 	}
 }
 
 TEST_CASE_METHOD(CharClassifyTest, "CharsOfClass") {
 	unsigned char buf[2] = {0, 0};
-	for (int i = 1; i < 256; i++) {
+	for (int i = 1; i < byteValues; i++) {
 		const CharacterClass thisClass = static_cast<CharacterClass>(i % 4);
 		buf[0] = i;
 		pcc->SetCharClasses(buf, thisClass);
@@ -99,24 +104,26 @@ TEST_CASE_METHOD(CharClassifyTest, "CharsOfClass") {
 		std::vector<unsigned char> buffer(size+1);
 		const unsigned char *pBuffer = buffer.data();
 		pcc->GetCharsOfClass(thisClass, buffer.data());
-		for (int i = 1; i < 256; i++) {
+		for (int i = 1; i < byteValues; i++) {
 			if (charClass[i] == thisClass) {
-				if (!memchr(pBuffer, i, size))
+				if (!memchr(pBuffer, i, size)) {
 					std::cerr
 					<< "Character " << i
 					<< " should be class " << GetClassName(thisClass)
 					<< ", but was not in GetCharsOfClass;"
 					<< " it is reported to be "
 					<< GetClassName(pcc->GetClass(i)) << std::endl;
+				}
 				REQUIRE(memchr(pBuffer, i, size));
 			} else {
-				if (memchr(pBuffer, i, size))
+				if (memchr(pBuffer, i, size)) {
 					std::cerr
 					<< "Character " << i
 					<< " should not be class " << GetClassName(thisClass)
 					<< ", but was in GetCharsOfClass"
 					<< " it is reported to be "
 					<< GetClassName(pcc->GetClass(i)) << std::endl;
+				}
 				REQUIRE_FALSE(memchr(pBuffer, i, size));
 			}
 		}

scintilla/test/unit/testDocument.cxx

@@ -8,6 +8,7 @@
 #include <stdexcept>
 #include <string_view>
 #include <vector>
+#include <map>
 #include <set>
 #include <optional>
 #include <algorithm>
@@ -33,6 +34,7 @@
 #include "Decoration.h"
 #include "CaseFolder.h"
 #include "Document.h"
+#include "UniConversion.h"
 
 #include "catch.hpp"
 
@@ -67,23 +69,6 @@ const Folding foldings1252[] = {
 	{0xd8, 0xf8, 0x07},
 };
 
-// Table of case folding for non-ASCII bytes in Windows Russian code page 1251
-const Folding foldings1251[] = {
-	{0x80, 0x90, 0x01},
-	{0x81, 0x83, 0x01},
-	{0x8a, 0x9a, 0x01},
-	{0x8c, 0x9c, 0x04},
-	{0xa1, 0xa2, 0x01},
-	{0xa3, 0xbc, 0x01},
-	{0xa5, 0xb4, 0x01},
-	{0xa8, 0xb8, 0x01},
-	{0xaa, 0xba, 0x01},
-	{0xaf, 0xbf, 0x01},
-	{0xb2, 0xb3, 0x01},
-	{0xbd, 0xbe, 0x01},
-	{0xc0, 0xe0, 0x20},
-};
-
 std::string ReadFile(const std::string &path) {
 	std::ifstream ifs(path, std::ios::binary);
 	std::string content((std::istreambuf_iterator<char>(ifs)),
@@ -956,6 +941,61 @@ TEST_CASE("SafeSegment") {
 		REQUIRE(text[length] == '\xf0');
 	}
 
+	SECTION("UTF-8 Character Fragments") {
+		// PositionCache breaks long texts into fixed length sub-strings that are passed to SafeSegment
+		// so the final character in the sub-string may be incomplete without all needed trail bytes.
+		// For UTF-8, SafeSegment first discards any final bytes that do not represent a valid character
+		// then discards the final whole character.
+
+		const DocPlus doc("", CpUtf8);
+
+		// break before last character after discarding incomplete last character: 0 trail byte
+		std::string_view text = "Japanese\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e\xc2";	// Invalid text as ends with start byte
+		size_t length = doc.document.SafeSegment(text);
+		REQUIRE(text[length - 1] == '\xac');
+		REQUIRE(text[length] == '\xe8');
+		REQUIRE(UTF8IsValid(text.substr(0, length)));
+
+		// break before last character after discarding incomplete last character: 1 trail byte and 2 needed
+		text = "Japanese\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e\xe6\x97";	// Invalid text as ends with only 1 trail byte
+		length = doc.document.SafeSegment(text);
+		REQUIRE(text[length - 1] == '\xac');
+		REQUIRE(text[length] == '\xe8');
+		REQUIRE(UTF8IsValid(text.substr(0, length)));
+	}
+
+	SECTION("UTF-8 Combining Characters") {
+		const DocPlus doc("", CpUtf8);
+
+		// There may be combining characters like accents and tone marks after the
+		// last letter in a sub-string and these may be included in the sub-string
+		// or follow it.
+		// Correct display requires that the combining characters are measured and
+		// drawn with the letter they follow. Thus the final letter and any
+		// following combining characters are discarded.
+
+		// A Thai text example with 8 characters, each taking 3 bytes:
+		// HO HIP, SARA AA, KHO KHAI, MAI THO, O ANG, MO MA, SARA UU, LO LING
+		// Most are letters (Lo) but 2 characters are modifiers (Mn):
+		// MAI THO is a tone mark and SARA UU is a vowel.
+		const std::string_view text = "\xe0\xb8\xab\xe0\xb8\xb2\xe0\xb8\x82\xe0\xb9\x89\xe0\xb8\xad\xe0\xb8\xa1\xe0\xb8\xb9\xe0\xb8\xa5";
+		REQUIRE(text.length() == 8 * 3);
+		size_t length = doc.document.SafeSegment(text);
+		REQUIRE(length == (8 - 1) * 3);	// Discard last character
+
+		// Remove last character (letter LO LING) then run again.
+		// Should skip past SARA UU combining vowel mark to discard letter MO MA and SARA UU.
+		const std::string_view textWithoutLoLing = text.substr(0, length);
+		length = doc.document.SafeSegment(textWithoutLoLing);
+		REQUIRE(length == (8 - 3) * 3);	// Discard 2 characters
+
+		// Remove last character SARA UU combining vowel mark then run again
+		// Final letter may have following combining mark so discard producing same text as previous step.
+		const std::string_view textWithoutSaraUu = text.substr(0, (8 - 2) * 3);
+		length = doc.document.SafeSegment(textWithoutSaraUu);
+		REQUIRE(length == (8 - 3) * 3);	// Discard 1 character
+	}
+
 	SECTION("DBCS Shift-JIS") {
 		const DocPlus doc("", 932);
 		// word and punctuation boundary in middle of text: single byte

scintilla/test/unit/testRESearch.cxx

@@ -40,10 +40,10 @@ public:
 	[[nodiscard]] Sci::Position Length() const noexcept {
 		return s.length();
 	}
-	char CharAt(Sci::Position index) const override {
+	[[nodiscard]] char CharAt(Sci::Position index) const override {
 		return s.at(index);
 	}
-	Sci::Position MovePositionOutsideChar(Sci::Position pos, [[maybe_unused]] Sci::Position moveDir) const noexcept override {
+	[[nodiscard]] Sci::Position MovePositionOutsideChar(Sci::Position pos, [[maybe_unused]] Sci::Position moveDir) const noexcept override {
 		return pos;
 	}
 	[[nodiscard]] std::string GetCharRange(Sci::Position position, Sci::Position lengthRetrieve) const {

scintilla/test/unit/testSelection.cxx

@@ -0,0 +1,295 @@
+/** @file testSelection.cxx
+ ** Unit Tests for Scintilla internal data structures
+ **/
+
+#include <cstdint>
+
+#include <stdexcept>
+#include <string_view>
+#include <vector>
+
+#include "Debugging.h"
+
+#include "Position.h"
+#include "Selection.h"
+
+#include "catch.hpp"
+
+using namespace Scintilla;
+using namespace Scintilla::Internal;
+
+// Test Selection.
+
+namespace {
+
+constexpr SelectionPosition invalid;
+constexpr SelectionPosition zero(0);
+constexpr SelectionRange rangeInvalid;
+constexpr SelectionRange rangeZero(0);
+
+}
+
+TEST_CASE("SelectionPosition") {
+
+	SECTION("SelectionPosition") {
+		SelectionPosition sel;
+		REQUIRE(sel.Position() == Sci::invalidPosition);
+		REQUIRE(sel.VirtualSpace() == 0);
+		REQUIRE(!sel.IsValid());
+		REQUIRE(sel.VirtualSpace() == 0);
+
+		REQUIRE(sel == invalid);
+		REQUIRE(sel != zero);
+		sel.Reset();
+		REQUIRE(sel != invalid);
+		REQUIRE(sel == zero);
+	}
+
+	SECTION("Comparison") {
+		constexpr SelectionPosition sel(2,3);
+		REQUIRE(sel > invalid);
+		REQUIRE(sel > zero);
+		REQUIRE(sel >= zero);
+		REQUIRE(zero < sel);
+		REQUIRE(zero <= sel);
+
+		SelectionPosition virtuous(0, 4);
+		REQUIRE(virtuous > zero);
+		REQUIRE(virtuous >= zero);
+		REQUIRE(zero < virtuous);
+		REQUIRE(zero <= virtuous);
+
+		REQUIRE(virtuous.Position() == 0);
+		REQUIRE(virtuous.VirtualSpace() == 4);
+
+		virtuous.SetPosition(1);	// Also resets virtualSpace
+		REQUIRE(virtuous.Position() == 1);
+		REQUIRE(virtuous.VirtualSpace() == 0);
+		virtuous.SetVirtualSpace(3);	// Does not reset position
+		REQUIRE(virtuous.Position() == 1);
+		REQUIRE(virtuous.VirtualSpace() == 3);
+	}
+
+	SECTION("Add") {
+		SelectionPosition sel(2,3);
+		sel.Add(1);
+		REQUIRE(sel.Position() == 3);
+		REQUIRE(sel.VirtualSpace() == 3);
+		sel.AddVirtualSpace(2);
+		REQUIRE(sel.Position() == 3);
+		REQUIRE(sel.VirtualSpace() == 5);
+	}
+
+	SECTION("MoveForInsertDelete") {
+		// There are multiple details implemented in MoveForInsertDelete that are supposed to
+		// move selections in a way that appears to be natural to a user.
+
+		SelectionPosition sel(2,3);
+		sel.MoveForInsertDelete(true, 0,1, false);
+		REQUIRE(sel == SelectionPosition(3,3));
+
+		// Converts a virtual space to real space
+		sel.MoveForInsertDelete(true, 3,1, false);
+		REQUIRE(sel == SelectionPosition(4,2));
+
+		// Deletion at position clears virtual space
+		sel.MoveForInsertDelete(false, 4,1, false);
+		REQUIRE(sel == SelectionPosition(4,0));
+
+		sel.MoveForInsertDelete(false, 3,1, false);
+		REQUIRE(sel == SelectionPosition(3,0));
+
+		// Insert at position with and without move for equal
+		sel.MoveForInsertDelete(true, 3, 1, false);
+		REQUIRE(sel == SelectionPosition(3, 0));
+		sel.MoveForInsertDelete(true, 3, 1, true);
+		REQUIRE(sel == SelectionPosition(4, 0));
+
+		// Deletion over the position moves to start of deletion
+		sel.MoveForInsertDelete(false, 2, 5, false);
+		REQUIRE(sel == SelectionPosition(2, 0));
+	}
+
+	SECTION("Serialization") {
+		// Conversion to/from string form
+
+		const std::string invalidString(invalid.ToString());
+		REQUIRE(invalidString == "-1");
+		const SelectionPosition invalidReturned(invalidString);
+		REQUIRE(invalidReturned == invalid);
+
+		const std::string zeroString(zero.ToString());
+		REQUIRE(zeroString == "0");
+		const SelectionPosition zeroReturned(zeroString);
+		REQUIRE(zeroReturned == zero);
+
+		const SelectionPosition virtue(2, 3);
+		const std::string virtueString(virtue.ToString());
+		REQUIRE(virtueString == "2v3");
+		const SelectionPosition virtueReturned(virtueString);
+		REQUIRE(virtueReturned == virtue);
+	}
+
+}
+
+TEST_CASE("SelectionSegment") {
+
+	SECTION("SelectionSegment") {
+		const SelectionSegment ss;
+		REQUIRE(ss.start == invalid);
+		REQUIRE(ss.end == invalid);
+	}
+
+}
+
+TEST_CASE("SelectionRange") {
+
+	SECTION("SelectionRange") {
+		const SelectionRange sr;
+		REQUIRE(sr.anchor == invalid);
+		REQUIRE(sr.caret == invalid);
+	}
+
+	SECTION("Serialization") {
+		// Conversion to/from string form
+
+		// Range from 1 to 2 with 3 virtual spaces
+		const SelectionRange range123(SelectionPosition(2, 3), SelectionPosition(1));
+		const std::string range123String(range123.ToString());
+		// Opposite order to constructor: from anchor to caret 
+		REQUIRE(range123String == "1-2v3");
+		const SelectionRange range123Returned(range123String);
+		REQUIRE(range123Returned == range123);
+	}
+
+	SECTION("Intersect") {
+		constexpr SelectionSegment segmentEmpty;
+
+		// Range from 1 to 2 with 3 virtual spaces
+		const SelectionRange range123(SelectionPosition(2, 3), SelectionPosition(1));
+		const SelectionSegment segment12(1, 2);
+		const SelectionSegment inside = range123.Intersect(segment12);
+		REQUIRE(inside == segment12);
+
+		const SelectionSegment segment121(SelectionPosition(1), SelectionPosition(2, 1));
+		const SelectionSegment withVirtual = range123.Intersect(segment121);
+		REQUIRE(withVirtual == segment121);
+
+		const SelectionSegment segment052(SelectionPosition(0), SelectionPosition(5, 2));
+		const SelectionSegment internal = range123.Intersect(segment052);	// All inside
+		REQUIRE(internal == range123.AsSegment());
+
+		const SelectionSegment wayOut(SelectionPosition(100), SelectionPosition(105, 2));
+		const SelectionSegment nothing = range123.Intersect(wayOut);
+		REQUIRE(nothing == segmentEmpty);
+
+		const SelectionSegment edge(1, 1);
+		const SelectionSegment nowt = range123.Intersect(edge);
+		REQUIRE(nowt == edge);
+
+		// (0, 1) and (1, 2v3) touch so intersection is a single position.
+		const SelectionSegment front(0, 1);
+		const SelectionSegment single(1, 1);
+		const SelectionSegment thin = range123.Intersect(front);
+		REQUIRE(thin == single);
+	}
+
+}
+
+TEST_CASE("Selection") {
+
+	SECTION("Selection") {
+		Selection sel;
+		
+		REQUIRE(sel.selType == Selection::SelTypes::stream);
+		REQUIRE(!sel.IsRectangular());
+		REQUIRE(sel.Count() == 1);
+		REQUIRE(sel.Main() == 0);
+
+		REQUIRE(sel.Range(0) == rangeZero);
+		REQUIRE(sel.RangeMain() == rangeZero);
+		REQUIRE(sel.Rectangular() == rangeInvalid);
+		REQUIRE(sel.Empty());
+	}
+
+	SECTION("Serialization") {
+		// Conversion to/from string form
+
+		// Range from 5 with 3 virtual spaces to 2
+		const SelectionRange range532(SelectionPosition(2), SelectionPosition(5, 3));
+		Selection selection;
+		selection.SetSelection(range532);
+		const std::string selectionString(selection.ToString());
+		// Opposite order to constructor: from anchor to caret 
+		REQUIRE(selectionString == "5v3-2");
+		const SelectionRange selectionReturned(selectionString);
+
+		REQUIRE(selection.selType == Selection::SelTypes::stream);
+		REQUIRE(!selection.IsRectangular());
+		REQUIRE(selection.Count() == 1);
+		REQUIRE(selection.Main() == 0);
+
+		REQUIRE(selection.Range(0) == range532);
+		REQUIRE(selection.RangeMain() == range532);
+		REQUIRE(selection.Rectangular() == rangeInvalid);
+		REQUIRE(!selection.Empty());
+	}
+
+	SECTION("SerializationMultiple") {
+		// Conversion to/from string form
+
+		// Range from 5 with 3 virtual spaces to 2
+		const SelectionRange range532(SelectionPosition(2), SelectionPosition(5, 3));
+		const SelectionRange range1(SelectionPosition(1));
+		Selection selection;
+		selection.SetSelection(range532);
+		selection.AddSelection(range1);
+		selection.SetMain(1);
+		const std::string selectionString(selection.ToString());
+		REQUIRE(selectionString == "5v3-2,1#1");
+		const SelectionRange selectionReturned(selectionString);
+
+		REQUIRE(selection.selType == Selection::SelTypes::stream);
+		REQUIRE(!selection.IsRectangular());
+		REQUIRE(selection.Count() == 2);
+		REQUIRE(selection.Main() == 1);
+
+		REQUIRE(selection.Range(0) == range532);
+		REQUIRE(selection.Range(1) == range1);
+		REQUIRE(selection.RangeMain() == range1);
+		REQUIRE(selection.Rectangular() == rangeInvalid);
+		REQUIRE(!selection.Empty());
+	}
+
+	SECTION("SerializationRectangular") {
+		// Conversion to/from string form
+
+		// Range from 5 with 3 virtual spaces to 2
+		const SelectionRange range532(SelectionPosition(2), SelectionPosition(5, 3));
+		
+		// Create a single-line rectangular selection
+		Selection selection;
+		selection.selType = Selection::SelTypes::rectangle;
+		selection.Rectangular() = range532;
+		// Set arbitrary realized range - inside editor ranges would be calculated from line layout
+		selection.SetSelection(rangeZero);
+
+		const std::string selectionString(selection.ToString());
+		REQUIRE(selectionString == "R5v3-2");
+		const Selection selectionReturned(selectionString);
+
+		REQUIRE(selection.selType == Selection::SelTypes::rectangle);
+		REQUIRE(selection.IsRectangular());
+		REQUIRE(selection.Count() == 1);
+		REQUIRE(selection.Main() == 0);
+
+		REQUIRE(selection.Range(0) == rangeZero);
+		REQUIRE(selection.RangeMain() == rangeZero);
+		REQUIRE(selection.Rectangular() == range532);
+
+		selection.selType = Selection::SelTypes::thin;
+		const std::string thinString(selection.ToString());
+		REQUIRE(thinString == "T5v3-2");
+	}
+
+}

scintilla/test/unit/testUniConversion.cxx

@@ -108,6 +108,11 @@ TEST_CASE("UniConversion") {
 		REQUIRE(UnicodeFromUTF8(s) == 0x10348);
 	}
 
+	SECTION("UnicodeFromUTF8 StringView") {
+		const unsigned char s[]="\xF0\x90\x8D\x88";
+		REQUIRE(UnicodeFromUTF8(s) == 0x10348);
+	}
+
 	// UTF16FromUTF8
 
 	SECTION("UTF16FromUTF8 ASCII") {

scintilla/test/unit/unitTest.cxx

@@ -4,33 +4,27 @@
 
 /*
     Currently tested:
-        SplitVector
-        Partitioning
-        RunStyles
-        ContractionState
+        CellBuffer
+        CharacterCategoryMap
         CharClassify
+        ContractionState
         Decoration
         DecorationList
-        CellBuffer
+        Document
+        Geometry
+        Partitioning
+        PerLine
+        RESearch
+        RunStyles
+        Selection
+        SplitVector
         UniConversion
 
     To do:
-        PerLine *
         Range
         StyledText
         CaseFolder ...
-        Document
-        RESearch
-        Selection
         Style
-
-        lexlib:
-        Accessor
-        LexAccessor
-        CharacterSet
-        OptionSet
-        PropSetSimple
-        StyleContext
 */
 
 #include <cstdio>