Update Scintilla to v5.3.7 & Lexilla to v5.2.7

Update scintilla with https://www.scintilla.org/scintilla537.zip

Release 5.3.7

    Released 22 September 2023.
    For GTK on macOS, fix popup window behaviour by setting type hints. Bug #2401.
    For GTK, fix assertion failure on some systems when an INDIC_SQUIGGLEPIXMAP drawn for a zero-width character.
    For Qt, allow parent window to handle context menu events by setting as ignored. Bug #2395.
    For Qt, fix potential crash when using IME with large amount of text selected.
    For Windows, fix building with non-English environment. Bug #2400.

and lexilla https://www.scintilla.org/lexilla527.zip

Release 5.2.7

    Released 22 September 2023.
    Fix building on Windows with non-English environment. Pull request #200.
    Bash: fix line continuation for comments and when multiple backslashes at line end. Issue #195.
    Bash: treat += as operator and, inside arithmetic expressions, treat ++ and -- as operators. Issue #197.
    Bash: improve backslash handling inside backquoted command substitution and fix $ at end of backtick expression. Issue #194.
    Bash: treat words that are similar to numbers but invalid wholly as identifiers. Issue #199.
    Bash: consistently handle '-' options at line start and after '|' as identifiers. Issue #202.
    Bash: handle '-' options differently in [ single ] and [[ double ]] bracket constructs. Issue #203.
    F#: improve speed of folding long lines. Issue #198.
    HTML: fix invalid entity at line end and terminate invalid entity before invalid character. Issue #192.

Fix #13991, fix #14062, close #14173

lexilla/.gitattributes

@@ -15,6 +15,7 @@
 **.html text
 **.bat text
 **.bsh text
+**.zsh text
 **.mak text
 **.def text
 **.manifest text
@@ -56,6 +57,7 @@
 **.mms text
 **.tex text
 **.fs text
+**.vh text
 **.vhd text
 **.x12 text
 **.yaml text

lexilla/.github/workflows/build-check-macos.yml

@@ -21,6 +21,10 @@ jobs:
           run: (cd test/unit && make DEBUG=1 CXX=${{matrix.cpp_compiler}} test)
         - name: Build Lexilla
           run: (cd src && make DEBUG=1 CXX=${{matrix.cpp_compiler}})
+        - uses: actions/upload-artifact@v3
+          with:
+              name: liblexilla.dylib
+              path: bin/liblexilla.dylib
         - name: Test lexing and folding
           run: (cd test && make DEBUG=1 CXX=${{matrix.cpp_compiler}} test)
         - name: CheckLexilla C Example

lexilla/.github/workflows/build-check-win32.yml

@@ -31,6 +31,10 @@ jobs:
               cd src
               nmake -f lexilla.mak DEBUG=1
               cd ..
+        - uses: actions/upload-artifact@v3
+          with:
+              name: lexilla.dll
+              path: bin/lexilla.dll
         - name: Test lexing and folding
           run: |
               cd test

lexilla/.github/workflows/build-check.yml

@@ -1,4 +1,4 @@
-name: "Build and check Lexilla"
+name: "Build and check Lexilla on Linux"
 
 on: [push]
 
@@ -21,6 +21,10 @@ jobs:
           run: (cd test/unit && make DEBUG=1 CXX=${{matrix.cpp_compiler}} test)
         - name: Build Lexilla
           run: (cd src && make DEBUG=1 CXX=${{matrix.cpp_compiler}})
+        - uses: actions/upload-artifact@v3
+          with:
+              name: liblexilla.so
+              path: bin/liblexilla.so
         - name: Test lexing and folding
           run: (cd test && make DEBUG=1 CXX=${{matrix.cpp_compiler}} test)
         - name: CheckLexilla C Example

lexilla/cppcheck.suppress

@@ -50,6 +50,7 @@ unreadVariable:lexilla/lexers/LexBaan.cxx
 variableScope:lexilla/lexers/LexBaan.cxx
 constParameterPointer:lexilla/lexers/LexBaan.cxx
 constParameterReference:lexilla/lexers/LexBash.cxx
+knownConditionTrueFalse:lexilla/lexers/LexBash.cxx
 variableScope:lexilla/lexers/LexBash.cxx
 constVariable:lexilla/lexers/LexBasic.cxx
 variableScope:lexilla/lexers/LexCmake.cxx

lexilla/doc/Lexilla.html

@@ -9,7 +9,7 @@
     <meta name="keywords" content="Scintilla, SciTE, Editing Component, Text Editor" />
     <meta name="Description"
     content="www.scintilla.org is the home of the Scintilla editing component and SciTE text editor application." />
-    <meta name="Date.Modified" content="20230726" />
+    <meta name="Date.Modified" content="20230922" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <style type="text/css">
         .logo {
@@ -61,8 +61,8 @@
           <font color="#FFCC99" size="4"> A library of language lexers for use with Scintilla</font>
         </td>
         <td width="40%" align="right">
-          <font color="#FFCC99" size="3">Release version 5.2.6<br />
+          <font color="#FFCC99" size="3">Release version 5.2.7<br />
-           Site last modified July 26 2023</font>
+           Site last modified September 22 2023</font>
         </td>
         <td width="20%">
           &nbsp;
@@ -77,11 +77,11 @@
       </tr>
     </table>
     <ul id="versionlist">
+      <li>Version 5.2.7 improves Bash, F#, and HTML.</li>
       <li>Version 5.2.6 improves Bash, Errorlist, HTML, Matlab, and Visual Prolog.</li>
       <li>Version 5.2.5 improves Bash, Batch, F#, and VB.</li>
       <li>Version 5.2.4 improves C++ and GDScript.</li>
       <li>Version 5.2.3 improves Makefile, Ruby, and YAML.</li>
-      <li>Version 5.2.2 improves C++, Matlab, Modula-3, Python, and X12.</li>
     </ul>
     <ul id="menu">
       <li id="remote1"><a href="https://www.scintilla.org/SciTEImage.html">Screenshot</a></li>

lexilla/doc/LexillaDownload.html

@@ -26,9 +26,9 @@
     <table bgcolor="#CCCCCC" width="100%" cellspacing="0" cellpadding="8" border="0">
       <tr>
         <td>
-          <font size="4"> <a href="https://www.scintilla.org/lexilla526.zip">
+          <font size="4"> <a href="https://www.scintilla.org/lexilla527.zip">
 	Windows</a>&nbsp;&nbsp;
-	<a href="https://www.scintilla.org/lexilla526.tgz">
+	<a href="https://www.scintilla.org/lexilla527.tgz">
           GTK/Linux</a>&nbsp;&nbsp;
 	</font>
         </td>
@@ -42,7 +42,7 @@
       containing very few restrictions.
     </p>
     <h3>
-       Release 5.2.6
+       Release 5.2.7
     </h3>
     <h4>
        Source Code
@@ -50,8 +50,8 @@
        The source code package contains all of the source code for Lexilla but no binary
 	executable code and is available in
        <ul>
-       <li><a href="https://www.scintilla.org/lexilla526.zip">zip format</a> (1.2M) commonly used on Windows</li>
+       <li><a href="https://www.scintilla.org/lexilla527.zip">zip format</a> (1.2M) commonly used on Windows</li>
-       <li><a href="https://www.scintilla.org/lexilla526.tgz">tgz format</a> (0.9M) commonly used on Linux and compatible operating systems</li>
+       <li><a href="https://www.scintilla.org/lexilla527.tgz">tgz format</a> (0.9M) commonly used on Linux and compatible operating systems</li>
        </ul>
        Instructions for building on both Windows and Linux are included in the readme file.
     <h4>

lexilla/doc/LexillaHistory.html

@@ -582,9 +582,54 @@
 	<td>Knut Leimbert</td>
       </tr><tr>
 	<td>German Aizek</td>
+	<td>Tsuyoshi Miyake</td>
     </tr>
     </table>
     <h2>Releases</h2>
+    <h3>
+       <a href="https://www.scintilla.org/lexilla527.zip">Release 5.2.7</a>
+    </h3>
+    <ul>
+	<li>
+	Released 22 September 2023.
+	</li>
+	<li>
+	Fix building on Windows with non-English environment.
+	<a href="https://github.com/ScintillaOrg/lexilla/pull/200">Pull request #200</a>.
+	</li>
+	<li>
+	Bash: fix line continuation for comments and when multiple backslashes at line end.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/195">Issue #195</a>.
+	</li>
+	<li>
+	Bash: treat  += as operator and, inside arithmetic expressions, treat ++ and -- as operators.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/197">Issue #197</a>.
+	</li>
+	<li>
+	Bash: improve backslash handling inside backquoted command substitution and fix $ at end of backtick expression.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/194">Issue #194</a>.
+	</li>
+	<li>
+	Bash: treat words that are similar to numbers but invalid wholly as identifiers.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/199">Issue #199</a>.
+	</li>
+	<li>
+	Bash: consistently handle '-' options at line start and after '|' as identifiers.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/202">Issue #202</a>.
+	</li>
+	<li>
+	Bash: handle '-' options differently in [ single ] and [[ double ]] bracket constructs.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/203">Issue #203</a>.
+	</li>
+	<li>
+	F#: improve speed of folding long lines.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/198">Issue #198</a>.
+	</li>
+	<li>
+	HTML: fix invalid entity at line end and terminate invalid entity before invalid character.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/192">Issue #192</a>.
+	</li>
+    </ul>
     <h3>
        <a href="https://www.scintilla.org/lexilla526.zip">Release 5.2.6</a>
     </h3>

lexilla/lexers/LexBash.cxx

@@ -56,17 +56,13 @@ enum class CmdState {
 	Body,
 	Start,
 	Word,
-	Test,
+	Test,			// test
+	SingleBracket,	// []
+	DoubleBracket,	// [[]]
 	Arithmetic,
 	Delimiter,
 };
 
-enum class TestExprType {
-	Test,			// test
-	DoubleBracket,	// [[]]
-	SingleBracket,	// []
-};
-
 enum class CommandSubstitution {
 	Backtick,
 	Inside,
@@ -81,7 +77,7 @@ enum class QuoteStyle {
 	String,			// ""
 	LString,		// $""
 	HereDoc,		// here document
-	Backtick,		// ``, $``
+	Backtick,		// ``
 	Parameter,		// ${}
 	Command,		// $()
 	CommandInside,	// $() with styling inside
@@ -172,6 +168,10 @@ bool IsCommentLine(Sci_Position line, LexAccessor &styler) {
 	return false;
 }
 
+constexpr bool StyleForceBacktrack(int state) noexcept {
+	return AnyOf(state, SCE_SH_CHARACTER, SCE_SH_STRING, SCE_SH_BACKTICKS, SCE_SH_HERE_Q, SCE_SH_PARAM);
+}
+
 struct OptionsBash {
 	bool fold = false;
 	bool foldComment = false;
@@ -180,6 +180,7 @@ struct OptionsBash {
 	bool stylingInsideBackticks = false;
 	bool stylingInsideParameter = false;
 	bool stylingInsideHeredoc = false;
+	bool nestedBackticks = true;
 	int commandSubstitution = static_cast<int>(CommandSubstitution::Backtick);
 	std::string specialParameter = BASH_SPECIAL_PARAMETER;
 
@@ -230,6 +231,9 @@ struct OptionSetBash : public OptionSet<OptionsBash> {
 			"1 highlighted inside. "
 			"2 highlighted inside with extra scope tracking.");
 
+		DefineProperty("lexer.bash.nested.backticks", &OptionsBash::nestedBackticks,
+			"Set this property to 0 to disable nested backquoted command substitution.");
+
 		DefineProperty("lexer.bash.special.parameter", &OptionsBash::specialParameter,
 			"Set shell (default is Bash) special parameters.");
 
@@ -267,8 +271,11 @@ class QuoteStackCls {	// Class to manage quote pairs that nest
 public:
 	int Depth = 0;
 	int State = SCE_SH_DEFAULT;
+	bool lineContinuation = false;
+	bool nestedBackticks = false;
 	CommandSubstitution commandSubstitution = CommandSubstitution::Backtick;
 	int insideCommand = 0;
+	unsigned backtickLevel = 0;
 	QuoteCls Current;
 	QuoteCls Stack[BASH_QUOTE_STACK_MAX];
 	const CharacterSet &setParamStart;
@@ -279,6 +286,9 @@ public:
 	void Start(int u, QuoteStyle s, int outer, CmdState state) noexcept {
 		if (Empty()) {
 			Current.Start(u, s, outer, state);
+			if (s == QuoteStyle::Backtick) {
+				++backtickLevel;
+			}
 		} else {
 			Push(u, s, outer, state);
 		}
@@ -290,13 +300,19 @@ public:
 		Stack[Depth] = Current;
 		Depth++;
 		Current.Start(u, s, outer, state);
+		if (s == QuoteStyle::Backtick) {
+			++backtickLevel;
+		}
 	}
 	void Pop() noexcept {
 		if (Depth == 0) {
 			Clear();
 			return;
 		}
-		if (insideCommand != 0) {
+		if (backtickLevel != 0 && Current.Style == QuoteStyle::Backtick) {
+			--backtickLevel;
+		}
+		if (insideCommand != 0 && Current.Style == QuoteStyle::CommandInside) {
 			insideCommand = 0;
 			for (int i = 0; i < Depth; i++) {
 				if (Stack[i].Style == QuoteStyle::CommandInside) {
@@ -312,6 +328,7 @@ public:
 		Depth = 0;
 		State = SCE_SH_DEFAULT;
 		insideCommand = 0;
+		backtickLevel = 0;
 		Current.Clear();
 	}
 	bool CountDown(StyleContext &sc, CmdState &cmdState) {
@@ -367,9 +384,6 @@ public:
 				// optimized to avoid track nested delimiter pairs
 				style = QuoteStyle::Literal;
 			}
-		} else if (sc.ch == '`') {	// $` seen in a configure script, valid?
-			style = QuoteStyle::Backtick;
-			sc.ChangeState(SCE_SH_BACKTICKS);
 		} else {
 			// scalar has no delimiter pair
 			if (!setParamStart.Contains(sc.ch)) {
@@ -386,6 +400,57 @@ public:
 			sc.Forward();
 		}
 	}
+	void Escape(StyleContext &sc) {
+		unsigned count = 1;
+		while (sc.chNext == '\\') {
+			++count;
+			sc.Forward();
+		}
+		bool escaped = count & 1U; // odd backslash escape next character
+		if (escaped && (sc.chNext == '\r' || sc.chNext == '\n')) {
+			lineContinuation = true;
+			if (sc.state == SCE_SH_IDENTIFIER) {
+				sc.SetState(SCE_SH_OPERATOR | insideCommand);
+			}
+			return;
+		}
+		if (backtickLevel > 0 && nestedBackticks) {
+			/*
+			for $k$ level substitution with $N$ backslashes:
+			* when $N/2^k$ is odd, following dollar is escaped.
+			* when $(N - 1)/2^k$ is even, following quote is escaped.
+			* when $N = n\times 2^{k + 1} - 1$, following backtick is escaped.
+			* when $N = n\times 2^{k + 1} + 2^k - 1$, following backtick starts inner substitution.
+			* when $N = m\times 2^k + 2^{k - 1} - 1$ and $k > 1$, following backtick ends current substitution.
+			*/
+			if (sc.chNext == '$') {
+				escaped = (count >> backtickLevel) & 1U;
+			} else if (sc.chNext == '\"' || sc.chNext == '\'') {
+				escaped = (((count - 1) >> backtickLevel) & 1U) == 0;
+			} else if (sc.chNext == '`' && escaped) {
+				unsigned mask = 1U << (backtickLevel + 1);
+				count += 1;
+				escaped = (count & (mask - 1)) == 0;
+				if (!escaped) {
+					unsigned remain = count - (mask >> 1U);
+					if (static_cast<int>(remain) >= 0 && (remain & (mask - 1)) == 0) {
+						escaped = true;
+						++backtickLevel;
+					} else if (backtickLevel > 1) {
+						mask >>= 1U;
+						remain = count - (mask >> 1U);
+						if (static_cast<int>(remain) >= 0 && (remain & (mask - 1)) == 0) {
+							escaped = true;
+							--backtickLevel;
+						}
+					}
+				}
+			}
+		}
+		if (escaped) {
+			sc.Forward();
+		}
+	}
 };
 
 const char styleSubable[] = { SCE_SH_IDENTIFIER, SCE_SH_SCALAR, 0 };
@@ -560,6 +625,7 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 	HereDocCls HereDoc;
 
 	QuoteStackCls QuoteStack(setParamStart);
+	QuoteStack.nestedBackticks = options.nestedBackticks;
 	QuoteStack.commandSubstitution = static_cast<CommandSubstitution>(options.commandSubstitution);
 
 	const WordClassifier &classifierIdentifiers = subStyles.Classifier(SCE_SH_IDENTIFIER);
@@ -569,7 +635,6 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 	int digit = 0;
 	const Sci_PositionU endPos = startPos + length;
 	CmdState cmdState = CmdState::Start;
-	TestExprType testExprType = TestExprType::Test;
 	LexAccessor styler(pAccess);
 
 	// Always backtracks to the start of a line that is not a continuation
@@ -591,33 +656,26 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 
 		// handle line continuation, updates per-line stored state
 		if (sc.atLineStart) {
-			if (MaskCommand(sc.state) == SCE_SH_STRING
+			CmdState state = CmdState::Body;	// force backtrack while retaining cmdState
-			 || MaskCommand(sc.state) == SCE_SH_BACKTICKS
+			if (!StyleForceBacktrack(MaskCommand(sc.state))) {
-			 || MaskCommand(sc.state) == SCE_SH_CHARACTER
-			 || MaskCommand(sc.state) == SCE_SH_HERE_Q
-			 || MaskCommand(sc.state) == SCE_SH_COMMENTLINE
-			 || MaskCommand(sc.state) == SCE_SH_PARAM) {
-				// force backtrack while retaining cmdState
-				styler.SetLineState(sc.currentLine, static_cast<int>(CmdState::Body));
-			} else {
-				if (sc.currentLine > 0) {
-					if ((sc.GetRelative(-3) == '\\' && sc.GetRelative(-2) == '\r' && sc.chPrev == '\n')
-					 || sc.GetRelative(-2) == '\\') {	// handle '\' line continuation
 				// retain last line's state
-					} else
+				// arithmetic expression and double bracket test can span multiline without line continuation
+				if (!QuoteStack.lineContinuation && !AnyOf(cmdState, CmdState::DoubleBracket, CmdState::Arithmetic)) {
 					cmdState = CmdState::Start;
 				}
-				// force backtrack when nesting
+				if (QuoteStack.Empty()) {	// force backtrack when nesting
-				const CmdState state = QuoteStack.Empty() ? cmdState : CmdState::Body;
+					state = cmdState;
-				styler.SetLineState(sc.currentLine, static_cast<int>(state));
 				}
 			}
+			QuoteStack.lineContinuation = false;
+			styler.SetLineState(sc.currentLine, static_cast<int>(state));
+		}
 
 		// controls change of cmdState at the end of a non-whitespace element
 		// states Body|Test|Arithmetic persist until the end of a command segment
 		// state Word persist, but ends with 'in' or 'do' construct keywords
 		CmdState cmdStateNew = CmdState::Body;
-		if (cmdState == CmdState::Test || cmdState == CmdState::Arithmetic || cmdState == CmdState::Word)
+		if (cmdState >= CmdState::Word && cmdState <= CmdState::Arithmetic)
 			cmdStateNew = cmdState;
 		const int stylePrev = MaskCommand(sc.state);
 		const int insideCommand = QuoteStack.insideCommand;
@@ -633,7 +691,7 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 				break;
 			case SCE_SH_WORD:
 				// "." never used in Bash variable names but used in file names
-				if (!setWord.Contains(sc.ch)) {
+				if (!setWord.Contains(sc.ch) || sc.Match('+', '=') || sc.Match('.', '.')) {
 					char s[500];
 					sc.GetCurrent(s, sizeof(s));
 					int identifierStyle = SCE_SH_IDENTIFIER | insideCommand;
@@ -661,7 +719,6 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 					if (strcmp(s, "test") == 0) {
 						if (cmdState == CmdState::Start && keywordEnds) {
 							cmdStateNew = CmdState::Test;
-							testExprType = TestExprType::Test;
 						} else
 							sc.ChangeState(identifierStyle);
 					}
@@ -681,7 +738,8 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 					}
 					// disambiguate option items and file test operators
 					else if (s[0] == '-') {
-						if (cmdState != CmdState::Test || !IsTestOperator(s, setSingleCharOp))
+						if (!AnyOf(cmdState, CmdState::Test, CmdState::SingleBracket, CmdState::DoubleBracket)
+							  || !keywordEnds || !IsTestOperator(s, setSingleCharOp))
 							sc.ChangeState(identifierStyle);
 					}
 					// disambiguate keywords and identifiers
@@ -693,7 +751,7 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 				}
 				break;
 			case SCE_SH_IDENTIFIER:
-				if (sc.chPrev == '\\' || !setWord.Contains(sc.ch) ||
+				if (!setWord.Contains(sc.ch) ||
 					  (cmdState == CmdState::Arithmetic && !setWordStart.Contains(sc.ch))) {
 					char s[500];
 					sc.GetCurrent(s, sizeof(s));
@@ -701,9 +759,6 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 					if (subStyle >= 0) {
 						sc.ChangeState(subStyle | insideCommand);
 					}
-					if (sc.chPrev == '\\') {	// for escaped chars
-						sc.Forward();
-					}
 					sc.SetState(SCE_SH_DEFAULT | insideCommand);
 				}
 				break;
@@ -755,11 +810,16 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 #endif
 				) {
 					sc.ChangeState(SCE_SH_ERROR | insideCommand);
+				} else if (digit < 62 || digit == 63 || (cmdState != CmdState::Arithmetic &&
+					(sc.ch == '-' || (sc.ch == '.' && sc.chNext != '.')))) {
+					// current character is alpha numeric, underscore, hyphen or dot
+					sc.ChangeState(SCE_SH_IDENTIFIER | insideCommand);
+					break;
 				}
 				sc.SetState(SCE_SH_DEFAULT | insideCommand);
 				break;
 			case SCE_SH_COMMENTLINE:
-				if (sc.MatchLineEnd() && sc.chPrev != '\\') {
+				if (sc.MatchLineEnd()) {
 					sc.SetState(SCE_SH_DEFAULT | insideCommand);
 				}
 				break;
@@ -873,7 +933,7 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 			case SCE_SH_BACKTICKS:
 				if (sc.ch == '\\') {
 					if (QuoteStack.Current.Style != QuoteStyle::Literal)
-						sc.Forward();
+						QuoteStack.Escape(sc);
 				} else if (sc.ch == QuoteStack.Current.Down) {
 					if (QuoteStack.CountDown(sc, cmdState)) {
 						continue;
@@ -968,8 +1028,7 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 			if (sc.ch == '\\') {
 				// Bash can escape any non-newline as a literal
 				sc.SetState(SCE_SH_IDENTIFIER | insideCommand);
-				if (sc.chNext == '\r' || sc.chNext == '\n')
+				QuoteStack.Escape(sc);
-					sc.SetState(SCE_SH_OPERATOR | insideCommand);
 			} else if (IsADigit(sc.ch)) {
 				sc.SetState(SCE_SH_NUMBER | insideCommand);
 				numBase = BASH_BASE_DECIMAL;
@@ -980,13 +1039,11 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 					} else if (IsADigit(sc.chNext)) {
 #ifdef PEDANTIC_OCTAL
 						numBase = BASH_BASE_OCTAL;
-#else
-						numBase = BASH_BASE_HEX;
 #endif
 					}
 				}
 			} else if (setWordStart.Contains(sc.ch)) {
-				sc.SetState(SCE_SH_WORD | insideCommand);
+				sc.SetState(((cmdState == CmdState::Arithmetic)? SCE_SH_IDENTIFIER : SCE_SH_WORD) | insideCommand);
 			} else if (sc.ch == '#') {
 				if (stylePrev != SCE_SH_WORD && stylePrev != SCE_SH_IDENTIFIER &&
 					(sc.currentPos == 0 || setMetaCharacter.Contains(sc.chPrev))) {
@@ -1033,10 +1090,13 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 					HereDoc.Indent = false;
 				}
 			} else if (sc.ch == '-' && // test operator or short and long option
-					   (IsUpperOrLowerCase(sc.chNext) || sc.chNext == '-') &&
+					   cmdState != CmdState::Arithmetic &&
-					   IsASpace(sc.chPrev)) {
+					   sc.chPrev != '~' && !IsADigit(sc.chNext)) {
+				if (IsASpace(sc.chPrev) || setMetaCharacter.Contains(sc.chPrev)) {
 					sc.SetState(SCE_SH_WORD | insideCommand);
-				sc.Forward();
+				} else {
+					sc.SetState(SCE_SH_IDENTIFIER | insideCommand);
+				}
 			} else if (setBashOperator.Contains(sc.ch)) {
 				bool isCmdDelim = false;
 				sc.SetState(SCE_SH_OPERATOR | insideCommand);
@@ -1066,12 +1126,10 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 						cmdState = CmdState::Arithmetic;
 						sc.Forward();
 					} else if (sc.Match('[', '[') && IsASpace(sc.GetRelative(2))) {
-						cmdState = CmdState::Test;
+						cmdState = CmdState::DoubleBracket;
-						testExprType = TestExprType::DoubleBracket;
 						sc.Forward();
 					} else if (sc.ch == '[' && IsASpace(sc.chNext)) {
-						cmdState = CmdState::Test;
+						cmdState = CmdState::SingleBracket;
-						testExprType = TestExprType::SingleBracket;
 					}
 				}
 				// special state -- for ((x;y;z)) in ... looping
@@ -1080,11 +1138,8 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 					sc.Forward(2);
 					continue;
 				}
-				// handle command delimiters in command Start|Body|Word state, also Test if 'test'
+				// handle command delimiters in command Start|Body|Word state, also Test if 'test' or '[]'
-				if (cmdState == CmdState::Start
+				if (cmdState < CmdState::DoubleBracket) {
-				 || cmdState == CmdState::Body
-				 || cmdState == CmdState::Word
-				 || (cmdState == CmdState::Test && testExprType == TestExprType::Test)) {
 					char s[10];
 					s[0] = static_cast<char>(sc.ch);
 					if (setBashOperator.Contains(sc.chNext)) {
@@ -1108,12 +1163,12 @@ void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int
 				if (cmdState == CmdState::Arithmetic && sc.Match(')', ')')) {
 					cmdState = CmdState::Body;
 					sc.Forward();
-				} else if (cmdState == CmdState::Test && IsASpace(sc.chPrev)) {
+				} else if (sc.ch == ']' && IsASpace(sc.chPrev)) {
-					if (sc.Match(']', ']') && testExprType == TestExprType::DoubleBracket) {
+					if (cmdState == CmdState::SingleBracket) {
+						cmdState = CmdState::Body;
+					} else if (cmdState == CmdState::DoubleBracket && sc.chNext == ']') {
+						cmdState = CmdState::Body;
 						sc.Forward();
-						cmdState = CmdState::Body;
-					} else if (sc.ch == ']' && testExprType == TestExprType::SingleBracket) {
-						cmdState = CmdState::Body;
 					}
 				}
 			}

lexilla/lexers/LexCPP.cxx

@@ -326,50 +326,27 @@ public:
 
 // Options used for LexerCPP
 struct OptionsCPP {
-	bool stylingWithinPreprocessor;
+	bool stylingWithinPreprocessor = false;
-	bool identifiersAllowDollars;
+	bool identifiersAllowDollars = true;
-	bool trackPreprocessor;
+	bool trackPreprocessor = true;
-	bool updatePreprocessor;
+	bool updatePreprocessor = true;
-	bool verbatimStringsAllowEscapes;
+	bool verbatimStringsAllowEscapes = false;
-	bool triplequotedStrings;
+	bool triplequotedStrings = false;
-	bool hashquotedStrings;
+	bool hashquotedStrings = false;
-	bool backQuotedStrings;
+	bool backQuotedStrings = false;
-	bool escapeSequence;
+	bool escapeSequence = false;
-	bool fold;
+	bool fold = false;
-	bool foldSyntaxBased;
+	bool foldSyntaxBased = true;
-	bool foldComment;
+	bool foldComment = false;
-	bool foldCommentMultiline;
+	bool foldCommentMultiline = true;
-	bool foldCommentExplicit;
+	bool foldCommentExplicit = true;
 	std::string foldExplicitStart;
 	std::string foldExplicitEnd;
-	bool foldExplicitAnywhere;
+	bool foldExplicitAnywhere = false;
-	bool foldPreprocessor;
+	bool foldPreprocessor = false;
-	bool foldPreprocessorAtElse;
+	bool foldPreprocessorAtElse = false;
-	bool foldCompact;
+	bool foldCompact = false;
-	bool foldAtElse;
+	bool foldAtElse = false;
-	OptionsCPP() {
-		stylingWithinPreprocessor = false;
-		identifiersAllowDollars = true;
-		trackPreprocessor = true;
-		updatePreprocessor = true;
-		verbatimStringsAllowEscapes = false;
-		triplequotedStrings = false;
-		hashquotedStrings = false;
-		backQuotedStrings = false;
-		escapeSequence = false;
-		fold = false;
-		foldSyntaxBased = true;
-		foldComment = false;
-		foldCommentMultiline = true;
-		foldCommentExplicit = true;
-		foldExplicitStart = "";
-		foldExplicitEnd = "";
-		foldExplicitAnywhere = false;
-		foldPreprocessor = false;
-		foldPreprocessorAtElse = false;
-		foldCompact = false;
-		foldAtElse = false;
-	}
 };
 
 const char *const cppWordLists[] = {

lexilla/lexers/LexFSharp.cxx

@@ -672,25 +672,23 @@ void SCI_METHOD LexerFSharp::Fold(Sci_PositionU start, Sci_Position length, int
 		const int stylePrev = style;
 		const char ch = chNext;
 		const bool inLineComment = (stylePrev == SCE_FSHARP_COMMENTLINE);
-		const bool inOpenStatement = LineContains(styler, "open ", lineCurrent, SCE_FSHARP_KEYWORD);
 		style = styleNext;
 		styleNext = styler.StyleAt(currentPos + 1);
 		chNext = styler.SafeGetCharAt(currentPos + 1);
 
 		if (options.foldComment) {
-			if (options.foldCommentMultiLine && inLineComment && atEOL &&
+			if (options.foldCommentMultiLine && inLineComment && atEOL) {
-			    (lineCurrent > 0 || LineContains(styler, "//", lineNext, SCE_FSHARP_COMMENTLINE))) {
 				FoldLexicalGroup(styler, levelNext, lineCurrent, "//", SCE_FSHARP_COMMENTLINE);
 			}
 
 			if (options.foldCommentStream && style == SCE_FSHARP_COMMENT && !inLineComment) {
 				if (stylePrev != SCE_FSHARP_COMMENT ||
 				    (styler.Match(currentPos, "(*") &&
-				     !LineContains(styler, "*)", lineCurrent, SCE_FSHARP_COMMENT))) {
+				     !LineContains(styler, "*)", currentPos + 2, SCE_FSHARP_COMMENT))) {
 					levelNext++;
 				} else if ((styleNext != SCE_FSHARP_COMMENT ||
 					    ((styler.Match(currentPos, "*)") &&
-					      !LineContains(styler, "(*", lineCurrent, SCE_FSHARP_COMMENT)) &&
+					      !LineContains(styler, "(*", styler.LineStart(lineCurrent), SCE_FSHARP_COMMENT)) &&
 					     styler.GetLineState(lineCurrent - 1) > 0)) &&
 					   !atEOL) {
 					levelNext--;
@@ -706,7 +704,7 @@ void SCI_METHOD LexerFSharp::Fold(Sci_PositionU start, Sci_Position length, int
 			}
 		}
 
-		if (options.foldImports && inOpenStatement && atEOL) {
+		if (options.foldImports && styler.Match(currentPos, "open ") && styleNext == SCE_FSHARP_KEYWORD) {
 			FoldLexicalGroup(styler, levelNext, lineCurrent, "open ", SCE_FSHARP_KEYWORD);
 		}
 
@@ -744,7 +742,7 @@ void SCI_METHOD LexerFSharp::Fold(Sci_PositionU start, Sci_Position length, int
 bool LineContains(LexAccessor &styler, const char *word, const Sci_Position start, const int chAttr) {
 	bool found = false;
 	bool requireStyle = (chAttr > SCE_FSHARP_DEFAULT);
-	for (Sci_Position i = styler.LineStart(start); i < styler.LineStart(start + 1) - 1; i++) {
+	for (Sci_Position i = start; i < styler.LineStart(styler.GetLine(start) + 1) - 1; i++) {
 		if (styler.Match(i, word)) {
 			found = requireStyle ? styler.StyleAt(i) == chAttr : true;
 			break;
@@ -755,9 +753,9 @@ bool LineContains(LexAccessor &styler, const char *word, const Sci_Position star
 
 void FoldLexicalGroup(LexAccessor &styler, int &levelNext, const Sci_Position lineCurrent, const char *word,
 		      const int chAttr) {
-	const Sci_Position linePrev = lineCurrent - 1;
+	const Sci_Position linePrev = styler.LineStart(lineCurrent - 1);
-	const Sci_Position lineNext = lineCurrent + 1;
+	const Sci_Position lineNext = styler.LineStart(lineCurrent + 1);
-	const bool follows = LineContains(styler, word, linePrev, chAttr);
+	const bool follows = (lineCurrent > 0) && LineContains(styler, word, linePrev, chAttr);
 	const bool isFollowed = LineContains(styler, word, lineNext, chAttr);
 
 	if (isFollowed && !follows) {

lexilla/lexers/LexHTML.cxx

@@ -42,16 +42,16 @@ namespace {
 enum script_type { eScriptNone = 0, eScriptJS, eScriptVBS, eScriptPython, eScriptPHP, eScriptXML, eScriptSGML, eScriptSGMLblock, eScriptComment };
 enum script_mode { eHtml = 0, eNonHtmlScript, eNonHtmlPreProc, eNonHtmlScriptPreProc };
 
-bool IsAWordChar(const int ch) noexcept {
+constexpr bool IsAWordChar(int ch) noexcept {
-	return (ch < 0x80) && (isalnum(ch) || ch == '.' || ch == '_');
+	return IsAlphaNumeric(ch) || ch == '.' || ch == '_';
 }
 
-bool IsAWordStart(const int ch) noexcept {
+constexpr bool IsAWordStart(int ch) noexcept {
-	return (ch < 0x80) && (isalnum(ch) || ch == '_');
+	return IsAlphaNumeric(ch) || ch == '_';
 }
 
 bool IsOperator(int ch) noexcept {
-	if (IsASCII(ch) && isalnum(ch))
+	if (IsAlphaNumeric(ch))
 		return false;
 	// '.' left out as it is used to make up numbers
 	if (ch == '%' || ch == '^' || ch == '&' || ch == '*' ||
@@ -64,6 +64,10 @@ bool IsOperator(int ch) noexcept {
 	return false;
 }
 
+unsigned char SafeGetUnsignedCharAt(Accessor &styler, Sci_Position position, char chDefault = ' ') {
+	return styler.SafeGetCharAt(position, chDefault);
+}
+
 void GetTextSegment(Accessor &styler, Sci_PositionU start, Sci_PositionU end, char *s, size_t len) {
 	Sci_PositionU i = 0;
 	for (; (i < end - start + 1) && (i < len-1); i++) {
@@ -472,16 +476,16 @@ int StateForScript(script_type scriptLanguage) noexcept {
 	return Result;
 }
 
-bool issgmlwordchar(int ch) noexcept {
+constexpr bool issgmlwordchar(int ch) noexcept {
 	return !IsASCII(ch) ||
-		(isalnum(ch) || ch == '.' || ch == '_' || ch == ':' || ch == '!' || ch == '#' || ch == '[');
+		(IsAlphaNumeric(ch) || ch == '.' || ch == '_' || ch == ':' || ch == '!' || ch == '#' || ch == '[');
 }
 
-bool IsPhpWordStart(int ch) noexcept {
+constexpr bool IsPhpWordStart(int ch) noexcept {
-	return (IsASCII(ch) && (isalpha(ch) || (ch == '_'))) || (ch >= 0x7f);
+	return (IsUpperOrLowerCase(ch) || (ch == '_')) || (ch >= 0x7f);
 }
 
-bool IsPhpWordChar(int ch) noexcept {
+constexpr bool IsPhpWordChar(int ch) noexcept {
 	return IsADigit(ch) || IsPhpWordStart(ch);
 }
 
@@ -502,7 +506,7 @@ bool IsScriptCommentState(const int state) noexcept {
 		   state == SCE_HJA_COMMENTLINE || state == SCE_HB_COMMENTLINE || state == SCE_HBA_COMMENTLINE;
 }
 
-bool isLineEnd(int ch) noexcept {
+constexpr bool isLineEnd(int ch) noexcept {
 	return ch == '\r' || ch == '\n';
 }
 
@@ -1162,10 +1166,12 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 	const CharacterSet setAttributeContinue(CharacterSet::setAlphaNum, ".-_:!#/", true);
 	// TODO: also handle + and - (except if they're part of ++ or --) and return keywords
 	const CharacterSet setOKBeforeJSRE(CharacterSet::setNone, "([{=,:;!%^&*|?~");
+	// Only allow [A-Za-z0-9.#-_:] in entities
+	const CharacterSet setEntity(CharacterSet::setAlphaNum, ".#-_:");
 
 	int levelPrev = styler.LevelAt(lineCurrent) & SC_FOLDLEVELNUMBERMASK;
 	int levelCurrent = levelPrev;
-	int visibleChars = 0;
+	Sci_Position visibleChars = 0;
 	int lineStartVisibleChars = 0;
 
 	int chPrev = ' ';
@@ -1182,7 +1188,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 				break;
 		}
 		if (style == SCE_HJ_SYMBOLS) {
-			chPrevNonWhite = static_cast<unsigned char>(styler.SafeGetCharAt(back));
+			chPrevNonWhite = SafeGetUnsignedCharAt(styler, back);
 		}
 	}
 
@@ -1195,8 +1201,8 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 			state != SCE_HJ_COMMENTLINE && state != SCE_HJ_COMMENTDOC)
 			chPrevNonWhite = ch;
 		ch = static_cast<unsigned char>(styler[i]);
-		int chNext = static_cast<unsigned char>(styler.SafeGetCharAt(i + 1));
+		int chNext = SafeGetUnsignedCharAt(styler, i + 1);
-		const int chNext2 = static_cast<unsigned char>(styler.SafeGetCharAt(i + 2));
+		const int chNext2 = SafeGetUnsignedCharAt(styler, i + 2);
 
 		// Handle DBCS codepages
 		if (styler.IsLeadByte(static_cast<char>(ch))) {
@@ -1410,7 +1416,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 				levelCurrent++;
 			}
 			// should be better
-			ch = static_cast<unsigned char>(styler.SafeGetCharAt(i));
+			ch = SafeGetUnsignedCharAt(styler, i);
 			continue;
 		}
 
@@ -1447,14 +1453,14 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 			styler.ColourTo(i, SCE_H_ASP);
 			if (ch != '%' && ch != '$' && ch != '/') {
 				i += makoBlockType.length();
-				visibleChars += static_cast<int>(makoBlockType.length());
+				visibleChars += makoBlockType.length();
 				if (keywords4.InList(makoBlockType.c_str()))
 					styler.ColourTo(i, SCE_HP_WORD);
 				else
 					styler.ColourTo(i, SCE_H_TAGUNKNOWN);
 			}
 
-			ch = static_cast<unsigned char>(styler.SafeGetCharAt(i));
+			ch = SafeGetUnsignedCharAt(styler, i);
 			continue;
 		}
 
@@ -1472,7 +1478,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 			scriptLanguage = eScriptComment;
 			state = SCE_H_COMMENT;
 			styler.ColourTo(i, SCE_H_ASP);
-			ch = static_cast<unsigned char>(styler.SafeGetCharAt(i));
+			ch = SafeGetUnsignedCharAt(styler, i);
 			continue;
 		} else if (isDjango && state == SCE_H_COMMENT && (ch == '#' && chNext == '}')) {
 			styler.ColourTo(i - 1, StateToPrint);
@@ -1508,7 +1514,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 			scriptLanguage = eScriptPython;
 			styler.ColourTo(i, SCE_H_ASP);
 
-			ch = static_cast<unsigned char>(styler.SafeGetCharAt(i));
+			ch = SafeGetUnsignedCharAt(styler, i);
 			continue;
 		}
 
@@ -1547,7 +1553,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 			}
 			styler.ColourTo(i, SCE_H_ASP);
 			// should be better
-			ch = static_cast<unsigned char>(styler.SafeGetCharAt(i));
+			ch = SafeGetUnsignedCharAt(styler, i);
 			continue;
 		}
 
@@ -1731,7 +1737,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 					styler.ColourTo(i - 2, StateToPrint);
 				}
 				state = SCE_H_SGML_COMMENT;
-			} else if (IsASCII(ch) && isalpha(ch) && (chPrev == '%')) {
+			} else if (IsUpperOrLowerCase(ch) && (chPrev == '%')) {
 				styler.ColourTo(i - 2, StateToPrint);
 				state = SCE_H_SGML_ENTITY;
 			} else if (ch == '#') {
@@ -1790,13 +1796,13 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 					styler.ColourTo(i - 1, SCE_H_SGML_DEFAULT);
 				}
 				// find the length of the word
-				int size = 1;
+				Sci_Position size = 1;
-				while (setHTMLWord.Contains(static_cast<unsigned char>(styler.SafeGetCharAt(i + size))))
+				while (setHTMLWord.Contains(SafeGetUnsignedCharAt(styler, i + size)))
 					size++;
 				styler.ColourTo(i + size - 1, StateToPrint);
 				i += size - 1;
 				visibleChars += size - 1;
-				ch = static_cast<unsigned char>(styler.SafeGetCharAt(i));
+				ch = SafeGetUnsignedCharAt(styler, i);
 				if (scriptLanguage == eScriptSGMLblock) {
 					state = SCE_H_SGML_BLOCK_DEFAULT;
 				} else {
@@ -1850,9 +1856,9 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 			}
 			break;
 		case SCE_H_SGML_SPECIAL:
-			if (!(IsASCII(ch) && isupper(ch))) {
+			if (!IsUpperCase(ch)) {
 				styler.ColourTo(i - 1, StateToPrint);
-				if (isalnum(ch)) {
+				if (IsAlphaNumeric(ch)) {
 					state = SCE_H_SGML_ERROR;
 				} else {
 					state = SCE_H_SGML_DEFAULT;
@@ -1863,7 +1869,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 			if (ch == ';') {
 				styler.ColourTo(i, StateToPrint);
 				state = SCE_H_SGML_DEFAULT;
-			} else if (!(IsASCII(ch) && isalnum(ch)) && ch != '-' && ch != '.') {
+			} else if (!(IsAlphaNumeric(ch)) && ch != '-' && ch != '.') {
 				styler.ColourTo(i, SCE_H_SGML_ERROR);
 				state = SCE_H_SGML_DEFAULT;
 			}
@@ -1872,14 +1878,15 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 			if (ch == ';') {
 				styler.ColourTo(i, StateToPrint);
 				state = SCE_H_DEFAULT;
-			}
+			} else if (!setEntity.Contains(ch)) {
-			if (ch != '#' && !(IsASCII(ch) && isalnum(ch))	// Should check that '#' follows '&', but it is unlikely anyway...
-				&& ch != '.' && ch != '-' && ch != '_' && ch != ':') { // valid in XML
-				if (!IsASCII(ch))	// Possibly start of a multibyte character so don't allow this byte to be in entity style
 				styler.ColourTo(i-1, SCE_H_TAGUNKNOWN);
-				else
-					styler.ColourTo(i, SCE_H_TAGUNKNOWN);
 				state = SCE_H_DEFAULT;
+				if (!isLineEnd(ch)) {
+					// Retreat one byte so the character that is invalid inside entity
+					// may start something else like a tag.
+					--i;
+					continue;
+				}
 			}
 			break;
 		case SCE_H_TAGUNKNOWN:
@@ -2197,10 +2204,10 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 		case SCE_HJ_REGEX:
 			if (ch == '\r' || ch == '\n' || ch == '/') {
 				if (ch == '/') {
-					while (IsASCII(chNext) && islower(chNext)) {   // gobble regex flags
+					while (IsLowerCase(chNext)) {   // gobble regex flags
 						i++;
 						ch = chNext;
-						chNext = static_cast<unsigned char>(styler.SafeGetCharAt(i + 1));
+						chNext = SafeGetUnsignedCharAt(styler, i + 1);
 					}
 				}
 				styler.ColourTo(i, StateToPrint);
@@ -2210,7 +2217,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 				if (chNext == '\\' || chNext == '/') {
 					i++;
 					ch = chNext;
-					chNext = static_cast<unsigned char>(styler.SafeGetCharAt(i + 1));
+					chNext = SafeGetUnsignedCharAt(styler, i + 1);
 				}
 			}
 			break;
@@ -2298,7 +2305,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 					state = SCE_HP_TRIPLEDOUBLE;
 					ch = ' ';
 					chPrev = ' ';
-					chNext = static_cast<unsigned char>(styler.SafeGetCharAt(i + 1));
+					chNext = SafeGetUnsignedCharAt(styler, i + 1);
 				} else {
 					//					state = statePrintForState(SCE_HP_STRING,inScriptType);
 					state = SCE_HP_STRING;
@@ -2310,7 +2317,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 					state = SCE_HP_TRIPLE;
 					ch = ' ';
 					chPrev = ' ';
-					chNext = static_cast<unsigned char>(styler.SafeGetCharAt(i + 1));
+					chNext = SafeGetUnsignedCharAt(styler, i + 1);
 				} else {
 					state = SCE_HP_CHARACTER;
 				}
@@ -2336,7 +2343,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 						state = SCE_HP_TRIPLEDOUBLE;
 						ch = ' ';
 						chPrev = ' ';
-						chNext = static_cast<unsigned char>(styler.SafeGetCharAt(i + 1));
+						chNext = SafeGetUnsignedCharAt(styler, i + 1);
 					} else {
 						state = SCE_HP_STRING;
 					}
@@ -2346,7 +2353,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 						state = SCE_HP_TRIPLE;
 						ch = ' ';
 						chPrev = ' ';
-						chNext = static_cast<unsigned char>(styler.SafeGetCharAt(i + 1));
+						chNext = SafeGetUnsignedCharAt(styler, i + 1);
 					} else {
 						state = SCE_HP_CHARACTER;
 					}
@@ -2366,7 +2373,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 				if (chNext == '\"' || chNext == '\'' || chNext == '\\') {
 					i++;
 					ch = chNext;
-					chNext = static_cast<unsigned char>(styler.SafeGetCharAt(i + 1));
+					chNext = SafeGetUnsignedCharAt(styler, i + 1);
 				}
 			} else if (ch == '\"') {
 				styler.ColourTo(i, StateToPrint);
@@ -2378,7 +2385,7 @@ void SCI_METHOD LexerHTML::Lex(Sci_PositionU startPos, Sci_Position length, int
 				if (chNext == '\"' || chNext == '\'' || chNext == '\\') {
 					i++;
 					ch = chNext;
-					chNext = static_cast<unsigned char>(styler.SafeGetCharAt(i + 1));
+					chNext = SafeGetUnsignedCharAt(styler, i + 1);
 				}
 			} else if (ch == '\'') {
 				styler.ColourTo(i, StateToPrint);

lexilla/lexers/LexPython.cxx

@@ -268,35 +268,19 @@ bool IsMatchOrCaseIdentifier(const StyleContext &sc, Accessor &styler, const cha
 
 // Options used for LexerPython
 struct OptionsPython {
-	int whingeLevel;
+	int whingeLevel = 0;
-	bool base2or8Literals;
+	bool base2or8Literals = true;
-	bool stringsU;
+	bool stringsU = true;
-	bool stringsB;
+	bool stringsB = true;
-	bool stringsF;
+	bool stringsF = true;
-	bool stringsOverNewline;
+	bool stringsOverNewline = false;
-	bool keywords2NoSubIdentifiers;
+	bool keywords2NoSubIdentifiers = false;
-	bool fold;
+	bool fold = false;
-	bool foldQuotes;
+	bool foldQuotes = false;
-	bool foldCompact;
+	bool foldCompact = false;
-	bool unicodeIdentifiers;
+	bool unicodeIdentifiers = true;
-	int identifierAttributes;
+	int identifierAttributes = 0;
-	int decoratorAttributes;
+	int decoratorAttributes = 0;
-
-	OptionsPython() noexcept {
-		whingeLevel = 0;
-		base2or8Literals = true;
-		stringsU = true;
-		stringsB = true;
-		stringsF = true;
-		stringsOverNewline = false;
-		keywords2NoSubIdentifiers = false;
-		fold = false;
-		foldQuotes = false;
-		foldCompact = false;
-		unicodeIdentifiers = true;
-		identifierAttributes = 0;
-		decoratorAttributes = 0;
-	}
 
 	literalsAllowed AllowedLiterals() const noexcept {
 		literalsAllowed allowedLiterals = stringsU ? litU : litNone;

lexilla/lexers/LexRegistry.cxx

@@ -40,12 +40,8 @@ static const char *const RegistryWordListDesc[] = {
 };
 
 struct OptionsRegistry {
-	bool foldCompact;
+	bool foldCompact = false;
-	bool fold;
+	bool fold = false;
-	OptionsRegistry() {
-		foldCompact = false;
-		fold = false;
-	}
 };
 
 struct OptionSetRegistry : public OptionSet<OptionsRegistry> {

lexilla/lexers/LexVerilog.cxx

@@ -472,7 +472,7 @@ void SCI_METHOD LexerVerilog::Lex(Sci_PositionU startPos, Sci_Position length, i
 			}
 		}
 
-		if (sc.atLineEnd) {
+		if (sc.MatchLineEnd()) {
 			curLine++;
 			lineEndNext = styler.LineEnd(curLine);
 			vlls.Add(curLine, preproc);
@@ -513,7 +513,7 @@ void SCI_METHOD LexerVerilog::Lex(Sci_PositionU startPos, Sci_Position length, i
 			sc.SetState(state|activitySet);
 		}
 
-		const bool atLineEndBeforeSwitch = sc.atLineEnd;
+		const bool atLineEndBeforeSwitch = sc.MatchLineEnd();
 
 		// Determine if the current state should terminate.
 		switch (MaskActive(sc.state)) {
@@ -564,7 +564,7 @@ void SCI_METHOD LexerVerilog::Lex(Sci_PositionU startPos, Sci_Position length, i
 				}
 				break;
 			case SCE_V_PREPROCESSOR:
-				if (!IsAWordChar(sc.ch) || sc.atLineEnd) {
+				if (!IsAWordChar(sc.ch) || sc.MatchLineEnd()) {
 					sc.SetState(SCE_V_DEFAULT|activitySet);
 				}
 				break;
@@ -593,14 +593,16 @@ void SCI_METHOD LexerVerilog::Lex(Sci_PositionU startPos, Sci_Position length, i
 					}
 				} else if (sc.ch == '\"') {
 					sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
-				} else if (sc.atLineEnd) {
+				} else if (sc.MatchLineEnd()) {
 					sc.ChangeState(SCE_V_STRINGEOL|activitySet);
+					if (sc.Match('\r', '\n'))
+						sc.Forward();
 					sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
 				}
 				break;
 		}
 
-		if (sc.atLineEnd && !atLineEndBeforeSwitch) {
+		if (sc.MatchLineEnd() && !atLineEndBeforeSwitch) {
 			// State exit processing consumed characters up to end of line.
 			curLine++;
 			lineEndNext = styler.LineEnd(curLine);
@@ -618,7 +620,7 @@ void SCI_METHOD LexerVerilog::Lex(Sci_PositionU startPos, Sci_Position length, i
 				do {
 					sc.Forward();
 				} while ((sc.ch == ' ' || sc.ch == '\t') && sc.More());
-				if (sc.atLineEnd) {
+				if (sc.MatchLineEnd()) {
 					sc.SetState(SCE_V_DEFAULT|activitySet);
 					styler.SetLineState(curLine, lineState);
 				} else {

lexilla/lexlib/CatalogueModules.h

@@ -15,7 +15,7 @@ namespace Lexilla {
 class CatalogueModules {
 	std::vector<LexerModule *> lexerCatalogue;
 public:
-	const LexerModule *Find(int language) const {
+	const LexerModule *Find(int language) const noexcept {
 		for (const LexerModule *lm : lexerCatalogue) {
 			if (lm->GetLanguage() == language) {
 				return lm;
@@ -43,24 +43,23 @@ public:
 		lexerCatalogue.insert(lexerCatalogue.end(), modules);
 	}
 
-	unsigned int Count() const noexcept {
+	size_t Count() const noexcept {
-		return static_cast<unsigned int>(lexerCatalogue.size());
+		return lexerCatalogue.size();
 	}
 
-	const char *Name(unsigned int index) const noexcept {
+	const char *Name(size_t index) const noexcept {
-		if (index < static_cast<unsigned int>(lexerCatalogue.size())) {
+		if (index < lexerCatalogue.size()) {
 			return lexerCatalogue[index]->languageName;
-		} else {
+		}
 		return "";
 	}
-	}
 
-	LexerFactoryFunction Factory(unsigned int index) const noexcept {
+	LexerFactoryFunction Factory(size_t index) const noexcept {
 		// Works for object lexers but not for function lexers
 		return lexerCatalogue[index]->fnFactory;
 	}
 
-	Scintilla::ILexer5 *Create(unsigned int index) const {
+	Scintilla::ILexer5 *Create(size_t index) const {
 		const LexerModule *plm = lexerCatalogue[index];
 		if (!plm) {
 			return nullptr;

lexilla/lexlib/OptionSet.h

@@ -28,22 +28,22 @@ class OptionSet {
 		std::string value;
 		std::string description;
 		Option() :
-			opType(SC_TYPE_BOOLEAN), pb(0), description("") {
+			opType(SC_TYPE_BOOLEAN), pb(nullptr) {
 		}
-		Option(plcob pb_, std::string description_="") :
+		Option(plcob pb_, std::string_view description_="") :
 			opType(SC_TYPE_BOOLEAN), pb(pb_), description(description_) {
 		}
-		Option(plcoi pi_, std::string description_) :
+		Option(plcoi pi_, std::string_view description_) :
 			opType(SC_TYPE_INTEGER), pi(pi_), description(description_) {
 		}
-		Option(plcos ps_, std::string description_) :
+		Option(plcos ps_, std::string_view description_) :
 			opType(SC_TYPE_STRING), ps(ps_), description(description_) {
 		}
 		bool Set(T *base, const char *val) {
 			value = val;
 			switch (opType) {
 			case SC_TYPE_BOOLEAN: {
-					bool option = atoi(val) != 0;
+					const bool option = atoi(val) != 0;
 					if ((*base).*pb != option) {
 						(*base).*pb = option;
 						return true;
@@ -51,7 +51,7 @@ class OptionSet {
 					break;
 				}
 			case SC_TYPE_INTEGER: {
-					int option = atoi(val);
+					const int option = atoi(val);
 					if ((*base).*pi != option) {
 						(*base).*pi = option;
 						return true;
@@ -85,30 +85,30 @@ class OptionSet {
 		names += name;
 	}
 public:
-	void DefineProperty(const char *name, plcob pb, std::string description="") {
+	void DefineProperty(const char *name, plcob pb, std::string_view description="") {
 		nameToDef[name] = Option(pb, description);
 		AppendName(name);
 	}
-	void DefineProperty(const char *name, plcoi pi, std::string description="") {
+	void DefineProperty(const char *name, plcoi pi, std::string_view description="") {
 		nameToDef[name] = Option(pi, description);
 		AppendName(name);
 	}
-	void DefineProperty(const char *name, plcos ps, std::string description="") {
+	void DefineProperty(const char *name, plcos ps, std::string_view description="") {
 		nameToDef[name] = Option(ps, description);
 		AppendName(name);
 	}
 	const char *PropertyNames() const noexcept {
 		return names.c_str();
 	}
-	int PropertyType(const char *name) {
+	int PropertyType(const char *name) const {
-		typename OptionMap::iterator it = nameToDef.find(name);
+		typename OptionMap::const_iterator const it = nameToDef.find(name);
 		if (it != nameToDef.end()) {
 			return it->second.opType;
 		}
 		return SC_TYPE_BOOLEAN;
 	}
-	const char *DescribeProperty(const char *name) {
+	const char *DescribeProperty(const char *name) const {
-		typename OptionMap::iterator it = nameToDef.find(name);
+		typename OptionMap::const_iterator const it = nameToDef.find(name);
 		if (it != nameToDef.end()) {
 			return it->second.description.c_str();
 		}
@@ -116,15 +116,15 @@ public:
 	}
 
 	bool PropertySet(T *base, const char *name, const char *val) {
-		typename OptionMap::iterator it = nameToDef.find(name);
+		typename OptionMap::iterator const it = nameToDef.find(name);
 		if (it != nameToDef.end()) {
 			return it->second.Set(base, val);
 		}
 		return false;
 	}
 
-	const char *PropertyGet(const char *name) {
+	const char *PropertyGet(const char *name) const {
-		typename OptionMap::iterator it = nameToDef.find(name);
+		typename OptionMap::const_iterator const it = nameToDef.find(name);
 		if (it != nameToDef.end()) {
 			return it->second.Get();
 		}

lexilla/lexlib/StyleContext.cxx

@@ -40,6 +40,8 @@ StyleContext::StyleContext(Sci_PositionU startPos, Sci_PositionU length,
 	styler.StartAt(startPos /*, chMask*/);
 	styler.StartSegment(startPos);
 
+	chPrev = GetRelativeCharacter(-1);
+
 	// Variable width is now 0 so GetNextChar gets the char at currentPos into chNext/widthNext
 	GetNextChar();
 	ch = chNext;

lexilla/lexlib/SubStyles.h

@@ -51,7 +51,7 @@ public:
 	}
 
 	int ValueFor(std::string_view s) const {
-		WordStyleMap::const_iterator it = wordToStyle.find(s);
+		WordStyleMap::const_iterator const it = wordToStyle.find(s);
 		if (it != wordToStyle.end())
 			return it->second;
 		else
@@ -82,7 +82,7 @@ public:
 			while (*cpSpace && !(*cpSpace == ' ' || *cpSpace == '\t' || *cpSpace == '\r' || *cpSpace == '\n'))
 				cpSpace++;
 			if (cpSpace > identifiers) {
-				std::string word(identifiers, cpSpace - identifiers);
+				const std::string word(identifiers, cpSpace - identifiers);
 				wordToStyle[word] = style;
 			}
 			identifiers = cpSpace;

lexilla/src/Lexilla.cxx

@@ -327,7 +327,7 @@ extern "C" {
 
 EXPORT_FUNCTION int CALLING_CONVENTION GetLexerCount() {
 	AddEachLexer();
-	return catalogueLexilla.Count();
+	return static_cast<int>(catalogueLexilla.Count());
 }
 
 EXPORT_FUNCTION void CALLING_CONVENTION GetLexerName(unsigned int index, char *name, int buflength) {
@@ -346,7 +346,7 @@ EXPORT_FUNCTION LexerFactoryFunction CALLING_CONVENTION GetLexerFactory(unsigned
 
 EXPORT_FUNCTION Scintilla::ILexer5 * CALLING_CONVENTION CreateLexer(const char *name) {
 	AddEachLexer();
-	for (unsigned int i = 0; i < catalogueLexilla.Count(); i++) {
+	for (size_t i = 0; i < catalogueLexilla.Count(); i++) {
 		const char *lexerName = catalogueLexilla.Name(i);
 		if (0 == strcmp(lexerName, name)) {
 			return catalogueLexilla.Create(i);
@@ -360,9 +360,8 @@ EXPORT_FUNCTION const char * CALLING_CONVENTION LexerNameFromID(int identifier)
 	const LexerModule *pModule = catalogueLexilla.Find(identifier);
 	if (pModule) {
 		return pModule->languageName;
-	} else {
-		return nullptr;
 	}
+	return nullptr;
 }
 
 EXPORT_FUNCTION const char * CALLING_CONVENTION GetLibraryPropertyNames() {

lexilla/src/Lexilla.pro

@@ -17,6 +17,7 @@ SOURCES += \
 
 HEADERS  += \
     ../include/Lexilla.h \
+    $$files(../lexlib/*.h, false) \
     $$files(../lexers/*.h, false)
 
 INCLUDEPATH += ../include ../lexlib ../../scintilla/include

lexilla/src/Lexilla.ruleset

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RuleSet Name="Lexilla Rules" Description="A set of rules for Lexilla. Don't check for owner, not_null, gsl::at, simple array access." ToolsVersion="17.0">
+  <IncludeAll Action="Warning" />
+  <Rules AnalyzerId="Microsoft.Analyzers.NativeCodeAnalysis" RuleNamespace="Microsoft.Rules.Native">
+    <Rule Id="C26400" Action="None" />
+    <Rule Id="C26429" Action="None" />
+    <Rule Id="C26446" Action="None" />
+    <Rule Id="C26455" Action="None" />
+    <Rule Id="C26481" Action="None" />
+    <Rule Id="C26482" Action="None" />
+    <Rule Id="C26485" Action="None" />
+  </Rules>
+</RuleSet>

lexilla/src/Lexilla.vcxproj

@@ -68,6 +68,8 @@
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup>
     <LinkIncremental>false</LinkIncremental>
+    <EnableClangTidyCodeAnalysis>true</EnableClangTidyCodeAnalysis>
+    <CodeAnalysisRuleSet>Lexilla.ruleset</CodeAnalysisRuleSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
     <TargetName>lib$(ProjectName)</TargetName>
@@ -97,6 +99,7 @@
       <MinimalRebuild>false</MinimalRebuild>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
       <LanguageStandard>stdcpp17</LanguageStandard>
+      <AdditionalOptions>/source-charset:utf-8 %(AdditionalOptions)</AdditionalOptions>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>

lexilla/src/Lexilla/Info.plist

@@ -15,7 +15,7 @@
 	<key>CFBundlePackageType</key>
 	<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
 	<key>CFBundleShortVersionString</key>
-	<string>5.2.6</string>
+	<string>5.2.7</string>
 	<key>CFBundleVersion</key>
 	<string>$(CURRENT_PROJECT_VERSION)</string>
 	<key>NSHumanReadableCopyright</key>

lexilla/src/Lexilla/Lexilla.xcodeproj/project.pbxproj

@@ -853,7 +853,7 @@
 			buildSettings = {
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 5.2.6;
+				CURRENT_PROJECT_VERSION = 5.2.7;
 				DEAD_CODE_STRIPPING = YES;
 				DEVELOPMENT_TEAM = 4F446KW87E;
 				DYLIB_COMPATIBILITY_VERSION = 1;
@@ -880,7 +880,7 @@
 			buildSettings = {
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 5.2.6;
+				CURRENT_PROJECT_VERSION = 5.2.7;
 				DEAD_CODE_STRIPPING = YES;
 				DEVELOPMENT_TEAM = 4F446KW87E;
 				DYLIB_COMPATIBILITY_VERSION = 1;

lexilla/src/LexillaVersion.rc

@@ -4,8 +4,8 @@
 
 #include <windows.h>
 
-#define VERSION_LEXILLA "5.2.6"
+#define VERSION_LEXILLA "5.2.7"
-#define VERSION_WORDS 5, 2, 6, 0
+#define VERSION_WORDS 5, 2, 7, 0
 
 VS_VERSION_INFO VERSIONINFO
 FILEVERSION	VERSION_WORDS

lexilla/src/lexilla.mak

@@ -36,7 +36,7 @@ SUBSYSTEM=-SUBSYSTEM:WINDOWS,10.00
 !ENDIF
 
 CRTFLAGS=-D_CRT_SECURE_NO_DEPRECATE=1 $(ADD_DEFINE)
-CXXFLAGS=-Zi -TP -MP -W4 -EHsc -std:c++17 $(CRTFLAGS)
+CXXFLAGS=-Zi -TP -MP -W4 -EHsc -std:c++17 -utf-8 $(CRTFLAGS)
 CXXDEBUG=-Od -MTd -DDEBUG
 CXXNDEBUG=-O2 -MT -DNDEBUG -GL
 NAME=-Fo
@@ -197,9 +197,6 @@ LEX_OBJS=\
 	$(DIR_O)\LexVisualProlog.obj \
 	$(DIR_O)\LexX12.obj \
 	$(DIR_O)\LexYAML.obj \
-	$(DIR_O)\LexObjC.obj \
-	$(DIR_O)\LexSearchResult.obj \
-	$(DIR_O)\LexUser.obj
 
 #--Autogenerated -- end of automatically generated section
 

lexilla/test/TestDocument.cxx

@@ -123,7 +123,7 @@ void TestDocument::Set(std::string_view sv) {
 	if (lineStarts.back() != Length()) {
 		lineStarts.push_back(Length());
 	}
-	lineStates.resize(lineStarts.size());
+	lineStates.resize(lineStarts.size() + 1);
 	lineLevels.resize(lineStarts.size(), 0x400);
 }
 
@@ -271,7 +271,7 @@ Sci_Position SCI_METHOD TestDocument::GetRelativePosition(Sci_Position positionS
 	if (characterOffset < 0) {
 		while (characterOffset < 0) {
 			if (pos <= 0) {
-				return 0;
+				return -1;
 			}
 			unsigned char previousByte = text.at(pos - 1);
 			if (previousByte < 0x80) {
@@ -302,8 +302,8 @@ Sci_Position SCI_METHOD TestDocument::GetRelativePosition(Sci_Position positionS
 
 int SCI_METHOD TestDocument::GetCharacterAndWidth(Sci_Position position, Sci_Position *pWidth) const {
 	// TODO: invalid UTF-8
-	if (position >= Length()) {
+	if ((position < 0) || (position >= Length())) {
-		// Return NULs after document end
+		// Return NULs before document start and after document end
 		if (pWidth) {
 			*pWidth = 1;
 		}

lexilla/test/TestLexers.cxx

@@ -987,22 +987,24 @@ std::filesystem::path FindLexillaDirectory(std::filesystem::path startDirectory)
 
 
 
-int main() {
+int main(int argc, char **argv) {
 	bool success = false;
-	// TODO: Allow specifying the base directory through a command line argument
 	const std::filesystem::path baseDirectory = FindLexillaDirectory(std::filesystem::current_path());
 	if (!baseDirectory.empty()) {
-		const std::filesystem::path examplesDirectory = baseDirectory / "test" / "examples";
+#if !defined(LEXILLA_STATIC)
-#if defined(LEXILLA_STATIC)
-		success = AccessLexilla(examplesDirectory);
-#else
 		const std::filesystem::path sharedLibrary = baseDirectory / "bin" / LEXILLA_LIB;
-		if (Lexilla::Load(sharedLibrary.string())) {
+		if (!Lexilla::Load(sharedLibrary.string())) {
-			success = AccessLexilla(examplesDirectory);
-		} else {
 			std::cout << "Failed to load " << sharedLibrary << "\n";
+			return 1;	// Indicate failure
 		}
 #endif
+		std::filesystem::path examplesDirectory = baseDirectory / "test" / "examples";
+		for (int i = 1; i < argc; i++) {
+			if (argv[i][0] != '-') {
+				examplesDirectory = argv[i];
+			}
+		}
+		success = AccessLexilla(examplesDirectory);
 	}
 	return success ? 0 : 1;
 }

lexilla/test/examples/bash/197ArithmeticOperator.bsh

@@ -0,0 +1,16 @@
+hello="hello, "
+hello+=word
+echo $hello
+
+for ((i = 2; i > 0; i--)); do
+   echo postfix dec $i
+done
+for ((i = 2; i > 0; --i)); do
+   echo prefix dec $i
+done
+for ((i = 0; i < 2; i++)); do
+   echo postfix inc $i
+done
+for ((i = 0; i < 2; ++i)); do
+   echo prefix inc $i
+done

lexilla/test/examples/bash/197ArithmeticOperator.bsh.folded

@@ -0,0 +1,17 @@
+ 0 400   0   hello="hello, "
+ 0 400   0   hello+=word
+ 0 400   0   echo $hello
+ 1 400   0   
+ 2 400   0 + for ((i = 2; i > 0; i--)); do
+ 0 401   0 |    echo postfix dec $i
+ 0 401   0 | done
+ 2 400   0 + for ((i = 2; i > 0; --i)); do
+ 0 401   0 |    echo prefix dec $i
+ 0 401   0 | done
+ 2 400   0 + for ((i = 0; i < 2; i++)); do
+ 0 401   0 |    echo postfix inc $i
+ 0 401   0 | done
+ 2 400   0 + for ((i = 0; i < 2; ++i)); do
+ 0 401   0 |    echo prefix inc $i
+ 0 401   0 | done
+ 0 400   0   

lexilla/test/examples/bash/197ArithmeticOperator.bsh.styled

@@ -0,0 +1,16 @@
+{8}hello{7}={5}"hello, "{0}
+{8}hello{7}+={8}word{0}
+{4}echo{0} {9}$hello{0}
+
+{4}for{0} {7}(({8}i{0} {7}={0} {3}2{7};{0} {8}i{0} {7}>{0} {3}0{7};{0} {8}i{7}--));{0} {4}do{0}
+   {4}echo{0} {8}postfix{0} {8}dec{0} {9}$i{0}
+{4}done{0}
+{4}for{0} {7}(({8}i{0} {7}={0} {3}2{7};{0} {8}i{0} {7}>{0} {3}0{7};{0} {7}--{8}i{7}));{0} {4}do{0}
+   {4}echo{0} {8}prefix{0} {8}dec{0} {9}$i{0}
+{4}done{0}
+{4}for{0} {7}(({8}i{0} {7}={0} {3}0{7};{0} {8}i{0} {7}<{0} {3}2{7};{0} {8}i{7}++));{0} {4}do{0}
+   {4}echo{0} {8}postfix{0} {8}inc{0} {9}$i{0}
+{4}done{0}
+{4}for{0} {7}(({8}i{0} {7}={0} {3}0{7};{0} {8}i{0} {7}<{0} {3}2{7};{0} {7}++{8}i{7}));{0} {4}do{0}
+   {4}echo{0} {8}prefix{0} {8}inc{0} {9}$i{0}
+{4}done{0}

lexilla/test/examples/bash/199Numbers.bsh

@@ -0,0 +1,90 @@
+# Lexing numeric literals
+
+# From issue #199
+
+# UUIDs
+
+virsh start 61a6a312-86d3-458c-824a-fa0adc2bd22c
+virsh start 61969312-86d3-458c-8249-fa0adc2bd22c
+virsh restore /opt/61a6a312-86d3-458c-824a-fa0adc2bd22c-suspend
+
+# Git items
+
+git checkout 998d611b516b0e485803089ecd53fdf0ea707a8c
+
+git log --no-walk 0e2ba9c
+git log --no-walk rel-5-2-4-97-g7405d4e7
+
+# Arithmetic and character ranges
+
+declare -i a=1+1; echo $a
+[[ $a == [0-9] ]] && echo 1
+
+# Brace expansion
+
+for i in {1..10..2}; do
+	echo $i
+done
+for a in {A..Z..2}; do
+	echo $a
+done
+
+# From Kein-Hong Man
+
+#--------------------------------------------------------------------------
+# Bash number formats
+# (20070712)
+# Octal lexing relaxed to allow hex digits to avoid flagging unnecessary
+# and misleading number errors; radix-prefixed lexing behaviour is unchanged,
+# as those cases are uncommon (to get strict lexing, define PEDANTIC_OCTAL).
+
+# NOTE: Some people may want an entire non-number to be lexed in the normal
+# style and not as part-number part-normal. If the user thinks there is a
+# better case for the former, please lobby for it on the SF issue tracker.
+
+0123 0567	# octal good
+08 0789 077ABC	# octal bad (disabled 20070712, now lexed as numbers)
+066XYZ		# octal bad
+0xDEAD 0X1234	# hex good
+0xABCMNO 0XGHI	# hex bad
+
+# extended "[base#]n" format where base is between 2-64
+# digits range are 0-9a-zA-Z@_
+# if base <= 36, then alphabets are case insensitive
+# this style isn't likely in non-number code, so the lexer currently
+# opts to colour the error in red -- send feedback if this is too
+# intrusive; 'invalid octals' (but valid text) in red proved annoying...
+
+2#10101		# binary
+2#23456		# error (in red)
+8#0123456789AB	# error (in red)
+16#abcDEF123
+16#abcpqr	# bad
+64#xyzXYZ@_789	# full base-64
+99#xyzXYZ@_789	# error (in red; invalid base)
+111#xyzXYZ@_789	# error (in red; invalid base)
+
+567+0123*0xBCD	# with operators
+(4#0123-3#012)
+
+# 20070712:
+# Octal lexing relaxed to avoid marking some number sequences as octal
+# errors. This is because the elements or apps controlled by bash may
+# have a different view of numbers, so we avoid flagging unnecessary
+# (and misleading) number errors. Radix-prefixed number lexing is
+# unchanged, as those cases are uncommon (no feedback on it yet.)
+
+# In the following, red-flagged 'octals' should now be lexed as normal
+# numbers, allowing hex digits.
+
+# flightgear missing.sh
+scriptversion=2004-09-07.08
+
+# git t/t0000/basic.sh
+P=087704a96baf1c2d1c869a8b084481e121c88b5b
+
+# openssh config.guess
+    *:procnto*:*:* | *:QNX:[0123456789]*:*)
+
+# with hex digits, the following will still be an invalid number
+066XYZ

lexilla/test/examples/bash/199Numbers.bsh.folded

@@ -0,0 +1,91 @@
+ 0 400   0   # Lexing numeric literals
+ 1 400   0   
+ 0 400   0   # From issue #199
+ 1 400   0   
+ 0 400   0   # UUIDs
+ 1 400   0   
+ 0 400   0   virsh start 61a6a312-86d3-458c-824a-fa0adc2bd22c
+ 0 400   0   virsh start 61969312-86d3-458c-8249-fa0adc2bd22c
+ 0 400   0   virsh restore /opt/61a6a312-86d3-458c-824a-fa0adc2bd22c-suspend
+ 1 400   0   
+ 0 400   0   # Git items
+ 1 400   0   
+ 0 400   0   git checkout 998d611b516b0e485803089ecd53fdf0ea707a8c
+ 1 400   0   
+ 0 400   0   git log --no-walk 0e2ba9c
+ 0 400   0   git log --no-walk rel-5-2-4-97-g7405d4e7
+ 1 400   0   
+ 0 400   0   # Arithmetic and character ranges
+ 1 400   0   
+ 0 400   0   declare -i a=1+1; echo $a
+ 0 400   0   [[ $a == [0-9] ]] && echo 1
+ 1 400   0   
+ 0 400   0   # Brace expansion
+ 1 400   0   
+ 2 400   0 + for i in {1..10..2}; do
+ 0 401   0 | 	echo $i
+ 0 401   0 | done
+ 2 400   0 + for a in {A..Z..2}; do
+ 0 401   0 | 	echo $a
+ 0 401   0 | done
+ 1 400   0   
+ 0 400   0   # From Kein-Hong Man
+ 1 400   0   
+ 2 400   0 + #--------------------------------------------------------------------------
+ 0 401   0 | # Bash number formats
+ 0 401   0 | # (20070712)
+ 0 401   0 | # Octal lexing relaxed to allow hex digits to avoid flagging unnecessary
+ 0 401   0 | # and misleading number errors; radix-prefixed lexing behaviour is unchanged,
+ 0 401   0 | # as those cases are uncommon (to get strict lexing, define PEDANTIC_OCTAL).
+ 1 400   0   
+ 2 400   0 + # NOTE: Some people may want an entire non-number to be lexed in the normal
+ 0 401   0 | # style and not as part-number part-normal. If the user thinks there is a
+ 0 401   0 | # better case for the former, please lobby for it on the SF issue tracker.
+ 1 400   0   
+ 0 400   0   0123 0567	# octal good
+ 0 400   0   08 0789 077ABC	# octal bad (disabled 20070712, now lexed as numbers)
+ 0 400   0   066XYZ		# octal bad
+ 0 400   0   0xDEAD 0X1234	# hex good
+ 0 400   0   0xABCMNO 0XGHI	# hex bad
+ 1 400   0   
+ 2 400   0 + # extended "[base#]n" format where base is between 2-64
+ 0 401   0 | # digits range are 0-9a-zA-Z@_
+ 0 401   0 | # if base <= 36, then alphabets are case insensitive
+ 0 401   0 | # this style isn't likely in non-number code, so the lexer currently
+ 0 401   0 | # opts to colour the error in red -- send feedback if this is too
+ 0 401   0 | # intrusive; 'invalid octals' (but valid text) in red proved annoying...
+ 1 400   0   
+ 0 400   0   2#10101		# binary
+ 0 400   0   2#23456		# error (in red)
+ 0 400   0   8#0123456789AB	# error (in red)
+ 0 400   0   16#abcDEF123
+ 0 400   0   16#abcpqr	# bad
+ 0 400   0   64#xyzXYZ@_789	# full base-64
+ 0 400   0   99#xyzXYZ@_789	# error (in red; invalid base)
+ 0 400   0   111#xyzXYZ@_789	# error (in red; invalid base)
+ 1 400   0   
+ 0 400   0   567+0123*0xBCD	# with operators
+ 0 400   0   (4#0123-3#012)
+ 1 400   0   
+ 2 400   0 + # 20070712:
+ 0 401   0 | # Octal lexing relaxed to avoid marking some number sequences as octal
+ 0 401   0 | # errors. This is because the elements or apps controlled by bash may
+ 0 401   0 | # have a different view of numbers, so we avoid flagging unnecessary
+ 0 401   0 | # (and misleading) number errors. Radix-prefixed number lexing is
+ 0 401   0 | # unchanged, as those cases are uncommon (no feedback on it yet.)
+ 1 400   0   
+ 2 400   0 + # In the following, red-flagged 'octals' should now be lexed as normal
+ 0 401   0 | # numbers, allowing hex digits.
+ 1 400   0   
+ 0 400   0   # flightgear missing.sh
+ 0 400   0   scriptversion=2004-09-07.08
+ 1 400   0   
+ 0 400   0   # git t/t0000/basic.sh
+ 0 400   0   P=087704a96baf1c2d1c869a8b084481e121c88b5b
+ 1 400   0   
+ 0 400   0   # openssh config.guess
+ 0 400   0       *:procnto*:*:* | *:QNX:[0123456789]*:*)
+ 1 400   0   
+ 0 400   0   # with hex digits, the following will still be an invalid number
+ 0 400   0   066XYZ
+ 0 400   0   

lexilla/test/examples/bash/199Numbers.bsh.styled

@@ -0,0 +1,90 @@
+{2}# Lexing numeric literals{0}
+
+{2}# From issue #199{0}
+
+{2}# UUIDs{0}
+
+{8}virsh{0} {8}start{0} {8}61a6a312-86d3-458c-824a-fa0adc2bd22c{0}
+{8}virsh{0} {8}start{0} {8}61969312-86d3-458c-8249-fa0adc2bd22c{0}
+{8}virsh{0} {8}restore{0} {7}/{8}opt{7}/{8}61a6a312-86d3-458c-824a-fa0adc2bd22c-suspend{0}
+
+{2}# Git items{0}
+
+{8}git{0} {8}checkout{0} {8}998d611b516b0e485803089ecd53fdf0ea707a8c{0}
+
+{8}git{0} {8}log{0} {8}--no-walk{0} {8}0e2ba9c{0}
+{8}git{0} {8}log{0} {8}--no-walk{0} {8}rel-5-2-4-97-g7405d4e7{0}
+
+{2}# Arithmetic and character ranges{0}
+
+{8}declare{0} {8}-i{0} {8}a{7}={3}1{7}+{3}1{7};{0} {4}echo{0} {9}$a{0}
+{7}[[{0} {9}$a{0} {7}=={0} {7}[{8}0-9{7}]{0} {7}]]{0} {7}&&{0} {4}echo{0} {3}1{0}
+
+{2}# Brace expansion{0}
+
+{4}for{0} {8}i{0} {4}in{0} {7}{{3}1{7}..{3}10{7}..{3}2{7}};{0} {4}do{0}
+	{4}echo{0} {9}$i{0}
+{4}done{0}
+{4}for{0} {8}a{0} {4}in{0} {7}{{8}A{7}..{8}Z{7}..{3}2{7}};{0} {4}do{0}
+	{4}echo{0} {9}$a{0}
+{4}done{0}
+
+{2}# From Kein-Hong Man{0}
+
+{2}#--------------------------------------------------------------------------{0}
+{2}# Bash number formats{0}
+{2}# (20070712){0}
+{2}# Octal lexing relaxed to allow hex digits to avoid flagging unnecessary{0}
+{2}# and misleading number errors; radix-prefixed lexing behaviour is unchanged,{0}
+{2}# as those cases are uncommon (to get strict lexing, define PEDANTIC_OCTAL).{0}
+
+{2}# NOTE: Some people may want an entire non-number to be lexed in the normal{0}
+{2}# style and not as part-number part-normal. If the user thinks there is a{0}
+{2}# better case for the former, please lobby for it on the SF issue tracker.{0}
+
+{3}0123{0} {3}0567{0}	{2}# octal good{0}
+{3}08{0} {3}0789{0} {8}077ABC{0}	{2}# octal bad (disabled 20070712, now lexed as numbers){0}
+{8}066XYZ{0}		{2}# octal bad{0}
+{3}0xDEAD{0} {3}0X1234{0}	{2}# hex good{0}
+{8}0xABCMNO{0} {8}0XGHI{0}	{2}# hex bad{0}
+
+{2}# extended "[base#]n" format where base is between 2-64{0}
+{2}# digits range are 0-9a-zA-Z@_{0}
+{2}# if base <= 36, then alphabets are case insensitive{0}
+{2}# this style isn't likely in non-number code, so the lexer currently{0}
+{2}# opts to colour the error in red -- send feedback if this is too{0}
+{2}# intrusive; 'invalid octals' (but valid text) in red proved annoying...{0}
+
+{3}2#10101{0}		{2}# binary{0}
+{1}2#23456{0}		{2}# error (in red){0}
+{1}8#0123456789{8}AB{0}	{2}# error (in red){0}
+{3}16#abcDEF123{0}
+{8}16#abcpqr{0}	{2}# bad{0}
+{3}64#xyzXYZ@_789{0}	{2}# full base-64{0}
+{1}99{8}#xyzXYZ{7}@{8}_789{0}	{2}# error (in red; invalid base){0}
+{1}111{8}#xyzXYZ{7}@{8}_789{0}	{2}# error (in red; invalid base){0}
+
+{3}567{7}+{3}0123{7}*{3}0xBCD{0}	{2}# with operators{0}
+{8}(4#0123-3#012){0}
+
+{2}# 20070712:{0}
+{2}# Octal lexing relaxed to avoid marking some number sequences as octal{0}
+{2}# errors. This is because the elements or apps controlled by bash may{0}
+{2}# have a different view of numbers, so we avoid flagging unnecessary{0}
+{2}# (and misleading) number errors. Radix-prefixed number lexing is{0}
+{2}# unchanged, as those cases are uncommon (no feedback on it yet.){0}
+
+{2}# In the following, red-flagged 'octals' should now be lexed as normal{0}
+{2}# numbers, allowing hex digits.{0}
+
+{2}# flightgear missing.sh{0}
+{8}scriptversion{7}={8}2004-09-07.08{0}
+
+{2}# git t/t0000/basic.sh{0}
+{8}P{7}={8}087704a96baf1c2d1c869a8b084481e121c88b5b{0}
+
+{2}# openssh config.guess{0}
+    {7}*:{8}procnto{7}*:*:*{0} {7}|{0} {7}*:{8}QNX{7}:[{3}0123456789{7}]*:*){0}
+
+{2}# with hex digits, the following will still be an invalid number{0}
+{8}066XYZ{0}

lexilla/test/examples/bash/202LineStartOption.bsh

@@ -0,0 +1,41 @@
+-a
+#
+-b
+#
+
+declare -A optionSet=([--help]=0)
+for option in {-h,--help,--version,--verbose,-,--}; do
+case $option in
+	-h|--help)
+		optionSet[--help]=1
+		echo help: $option
+		;;
+	-*-version)
+		echo version: $option
+		;;
+	--)
+		echo stop
+		;;
+	-)
+		echo stdin
+		;;
+	-*[-a-zA-Z0-9])
+		echo other: $option
+		;;
+esac
+done
+
+option=--help
+[[ $option == *-h* ]] && echo $option=${optionSet[$option]}
+
+for gcc in gcc{,-1{4..0..-1}}; do
+	echo $gcc
+done
+
+for gcc in gcc{,{-14..-10}}; do
+	echo $gcc
+done
+
+# Tilde-refix ~
+~+/foo
+~-/foo

lexilla/test/examples/bash/202LineStartOption.bsh.folded

@@ -0,0 +1,42 @@
+ 0 400   0   -a
+ 0 400   0   #
+ 0 400   0   -b
+ 0 400   0   #
+ 1 400   0   
+ 0 400   0   declare -A optionSet=([--help]=0)
+ 2 400   0 + for option in {-h,--help,--version,--verbose,-,--}; do
+ 2 401   0 + case $option in
+ 0 402   0 | 	-h|--help)
+ 0 402   0 | 		optionSet[--help]=1
+ 0 402   0 | 		echo help: $option
+ 0 402   0 | 		;;
+ 0 402   0 | 	-*-version)
+ 0 402   0 | 		echo version: $option
+ 0 402   0 | 		;;
+ 0 402   0 | 	--)
+ 0 402   0 | 		echo stop
+ 0 402   0 | 		;;
+ 0 402   0 | 	-)
+ 0 402   0 | 		echo stdin
+ 0 402   0 | 		;;
+ 0 402   0 | 	-*[-a-zA-Z0-9])
+ 0 402   0 | 		echo other: $option
+ 0 402   0 | 		;;
+ 0 402   0 | esac
+ 0 401   0 | done
+ 1 400   0   
+ 0 400   0   option=--help
+ 0 400   0   [[ $option == *-h* ]] && echo $option=${optionSet[$option]}
+ 1 400   0   
+ 2 400   0 + for gcc in gcc{,-1{4..0..-1}}; do
+ 0 401   0 | 	echo $gcc
+ 0 401   0 | done
+ 1 400   0   
+ 2 400   0 + for gcc in gcc{,{-14..-10}}; do
+ 0 401   0 | 	echo $gcc
+ 0 401   0 | done
+ 1 400   0   
+ 0 400   0   # Tilde-refix ~
+ 0 400   0   ~+/foo
+ 0 400   0   ~-/foo
+ 0 400   0   

lexilla/test/examples/bash/202LineStartOption.bsh.styled

@@ -0,0 +1,41 @@
+{8}-a{0}
+{2}#{0}
+{8}-b{0}
+{2}#{0}
+
+{8}declare{0} {8}-A{0} {8}optionSet{7}=([{8}--help{7}]={3}0{7}){0}
+{4}for{0} {8}option{0} {4}in{0} {7}{{8}-h{7},{8}--help{7},{8}--version{7},{8}--verbose{7},{8}-{7},{8}--{7}};{0} {4}do{0}
+{4}case{0} {9}$option{0} {4}in{0}
+	{8}-h{7}|{8}--help{7}){0}
+		{8}optionSet{7}[{8}--help{7}]={3}1{0}
+		{4}echo{0} {8}help{7}:{0} {9}$option{0}
+		{7};;{0}
+	{8}-{7}*{8}-version{7}){0}
+		{4}echo{0} {8}version{7}:{0} {9}$option{0}
+		{7};;{0}
+	{8}--{7}){0}
+		{4}echo{0} {8}stop{0}
+		{7};;{0}
+	{8}-{7}){0}
+		{4}echo{0} {8}stdin{0}
+		{7};;{0}
+	{8}-{7}*[{8}-a-zA-Z0-9{7}]){0}
+		{4}echo{0} {8}other{7}:{0} {9}$option{0}
+		{7};;{0}
+{4}esac{0}
+{4}done{0}
+
+{8}option{7}={8}--help{0}
+{7}[[{0} {9}$option{0} {7}=={0} {7}*{8}-h{7}*{0} {7}]]{0} {7}&&{0} {4}echo{0} {9}$option{7}={10}${optionSet[$option]}{0}
+
+{4}for{0} {8}gcc{0} {4}in{0} {8}gcc{7}{,-{3}1{7}{{3}4{7}..{3}0{7}..-{3}1{7}}};{0} {4}do{0}
+	{4}echo{0} {9}$gcc{0}
+{4}done{0}
+
+{4}for{0} {8}gcc{0} {4}in{0} {8}gcc{7}{,{-{3}14{7}..-{3}10{7}}};{0} {4}do{0}
+	{4}echo{0} {9}$gcc{0}
+{4}done{0}
+
+{2}# Tilde-refix ~{0}
+{7}~+/{8}foo{0}
+{7}~-/{8}foo{0}

lexilla/test/examples/bash/203TestOption.bsh

@@ -0,0 +1,24 @@
+[[ $1 == -e* ]] && echo e
+
+if [[ -d /usr/bin &&
+	-e /usr/bin/bash ]]; then
+	echo find bash
+fi
+
+if [[ -d /usr/bin &&	-e /usr/bin/bash ]]; then
+	echo find bash
+fi
+
+if [ -d /usr/bin && -e /usr/bin/bash ]; then
+	echo find bash
+fi
+
+if [ -d /usr/bin &&
+	-e /usr/bin/bash ]; then
+	echo find bash
+fi
+
+if [ -d /usr/bin && \
+	-e /usr/bin/bash ]; then
+	echo find bash
+fi

lexilla/test/examples/bash/203TestOption.bsh.folded

@@ -0,0 +1,25 @@
+ 0 400   0   [[ $1 == -e* ]] && echo e
+ 1 400   0   
+ 2 400   0 + if [[ -d /usr/bin &&
+ 0 401   0 | 	-e /usr/bin/bash ]]; then
+ 0 401   0 | 	echo find bash
+ 0 401   0 | fi
+ 1 400   0   
+ 2 400   0 + if [[ -d /usr/bin &&	-e /usr/bin/bash ]]; then
+ 0 401   0 | 	echo find bash
+ 0 401   0 | fi
+ 1 400   0   
+ 2 400   0 + if [ -d /usr/bin && -e /usr/bin/bash ]; then
+ 0 401   0 | 	echo find bash
+ 0 401   0 | fi
+ 1 400   0   
+ 2 400   0 + if [ -d /usr/bin &&
+ 0 401   0 | 	-e /usr/bin/bash ]; then
+ 0 401   0 | 	echo find bash
+ 0 401   0 | fi
+ 1 400   0   
+ 2 400   0 + if [ -d /usr/bin && \
+ 0 401   0 | 	-e /usr/bin/bash ]; then
+ 0 401   0 | 	echo find bash
+ 0 401   0 | fi
+ 0 400   0   

lexilla/test/examples/bash/203TestOption.bsh.styled

@@ -0,0 +1,24 @@
+{7}[[{0} {9}$1{0} {7}=={0} {8}-e{7}*{0} {7}]]{0} {7}&&{0} {4}echo{0} {8}e{0}
+
+{4}if{0} {7}[[{0} {4}-d{0} {7}/{8}usr{7}/{8}bin{0} {7}&&{0}
+	{4}-e{0} {7}/{8}usr{7}/{8}bin{7}/{8}bash{0} {7}]];{0} {4}then{0}
+	{4}echo{0} {8}find{0} {8}bash{0}
+{4}fi{0}
+
+{4}if{0} {7}[[{0} {4}-d{0} {7}/{8}usr{7}/{8}bin{0} {7}&&{0}	{4}-e{0} {7}/{8}usr{7}/{8}bin{7}/{8}bash{0} {7}]];{0} {4}then{0}
+	{4}echo{0} {8}find{0} {8}bash{0}
+{4}fi{0}
+
+{4}if{0} {7}[{0} {4}-d{0} {7}/{8}usr{7}/{8}bin{0} {7}&&{0} {8}-e{0} {7}/{8}usr{7}/{8}bin{7}/{8}bash{0} {7}];{0} {4}then{0}
+	{4}echo{0} {8}find{0} {8}bash{0}
+{4}fi{0}
+
+{4}if{0} {7}[{0} {4}-d{0} {7}/{8}usr{7}/{8}bin{0} {7}&&{0}
+	{8}-e{0} {7}/{8}usr{7}/{8}bin{7}/{8}bash{0} {7}];{0} {4}then{0}
+	{4}echo{0} {8}find{0} {8}bash{0}
+{4}fi{0}
+
+{4}if{0} {7}[{0} {4}-d{0} {7}/{8}usr{7}/{8}bin{0} {7}&&{0} {7}\{0}
+	{8}-e{0} {7}/{8}usr{7}/{8}bin{7}/{8}bash{0} {7}];{0} {4}then{0}
+	{4}echo{0} {8}find{0} {8}bash{0}
+{4}fi{0}

lexilla/test/examples/bash/AllStyles.bsh

@@ -48,6 +48,7 @@ $"string"
 $'str'
 
 # back-ticks=11
+`ls`
 $`ls`
 $(ls)
 

lexilla/test/examples/bash/AllStyles.bsh.folded

@@ -48,6 +48,7 @@
  0 400   0   $'str'
  1 400   0   
  0 400   0   # back-ticks=11
+ 0 400   0   `ls`
  0 400   0   $`ls`
  0 400   0   $(ls)
  1 400   0   

lexilla/test/examples/bash/AllStyles.bsh.styled

@@ -48,7 +48,8 @@
 {5}$'str'{0}
 
 {2}# back-ticks=11{0}
-{11}$`ls`{0}
+{11}`ls`{0}
+${11}`ls`{0}
 {11}$(ls){0}
 
 {2}# Use substyles{0}

lexilla/test/examples/bash/SciTE.properties

@@ -1,7 +1,7 @@
-lexer.*.bsh=bash
+lexer.*.bsh;*.zsh=bash
 fold=1
 fold.comment=1
-keywords.*.bsh=case cat do done echo else esac exit export fi find for if in pwd set then while
+keywords.*.bsh;*.zsh=case cat do done echo else esac exit export fi find for if in print pwd set setopt then while
 
 # Can use substyles for identifiers and scalars
 substyles.bash.8=1

lexilla/test/examples/bash/continuation.bsh

@@ -0,0 +1,15 @@
+# Tests for line continuation.
+# Issue #195.
+
+#backslash1\
+echo 1
+#backslash2\\
+echo 2
+
+if [ 1 ]; then
+    backslash1=A\
+fi
+    backslash2=B\\
+fi
+
+echo $backslash1, $backslash2

lexilla/test/examples/bash/continuation.bsh.folded

@@ -0,0 +1,16 @@
+ 2 400   0 + # Tests for line continuation.
+ 0 401   0 | # Issue #195.
+ 1 400   0   
+ 0 400   0   #backslash1\
+ 0 400   0   echo 1
+ 0 400   0   #backslash2\\
+ 0 400   0   echo 2
+ 1 400   0   
+ 2 400   0 + if [ 1 ]; then
+ 0 401   0 |     backslash1=A\
+ 0 401   0 | fi
+ 0 401   0 |     backslash2=B\\
+ 0 401   0 | fi
+ 1 400   0   
+ 0 400   0   echo $backslash1, $backslash2
+ 0 400   0   

lexilla/test/examples/bash/continuation.bsh.styled

@@ -0,0 +1,15 @@
+{2}# Tests for line continuation.{0}
+{2}# Issue #195.{0}
+
+{2}#backslash1\{0}
+{4}echo{0} {3}1{0}
+{2}#backslash2\\{0}
+{4}echo{0} {3}2{0}
+
+{4}if{0} {7}[{0} {3}1{0} {7}];{0} {4}then{0}
+    {8}backslash1{7}={8}A{7}\{0}
+{8}fi{0}
+    {8}backslash2{7}={8}B\\{0}
+{4}fi{0}
+
+{4}echo{0} {9}$backslash1{7},{0} {9}$backslash2{0}

lexilla/test/examples/bash/hash.zsh

@@ -0,0 +1,51 @@
+#!/bin/zsh
+# Tests for zsh extensions
+# Can be executed by zsh with reasonable results
+# Some of these were implemented by commit [87286d] for Scintilla bug #1794
+# https://zsh.sourceforge.io/Doc/Release/Expansion.html
+
+# Where # does not start a comment
+
+
+## Formatting base
+print $(( [#8] y = 33 ))
+print $(( [##8] 32767 ))
+
+# Formatting base and grouping
+print $(( [#16_4] 65536 ** 2 ))
+
+
+## Character values
+print $(( ##T+0 ))
+print $(( ##^G+0 ))
+# Failure: does not work when - included for bindkey syntax. \M-\C-x means Meta+Ctrl+x.
+print $(( ##\M-\C-x+0 ))
+
+# Value of first character of variable in expression
+var=Tree
+print $(( #var+0 ))
+
+
+## Extended glob
+setopt extended_glob
+
+# # is similar to *, ## similar to +
+echo [A-Za-z]#.bsh
+echo [A-Za-z]##.bsh
+
+# 13 character file names
+echo **/[a-zA-Z.](#c13)
+# 13-15 character file names
+echo **/[a-zA-Z.](#c13,15)
+
+
+## Glob flag
+
+# i=case-insensitive
+echo (#i)a*
+
+# b=back-references
+foo="a_string_with_a_message"
+if [[ $foo = (a|an)_(#b)(*) ]]; then
+  print ${foo[$mbegin[1],$mend[1]]}
+fi

lexilla/test/examples/bash/hash.zsh.folded

@@ -0,0 +1,52 @@
+ 2 400   0 + #!/bin/zsh
+ 0 401   0 | # Tests for zsh extensions
+ 0 401   0 | # Can be executed by zsh with reasonable results
+ 0 401   0 | # Some of these were implemented by commit [87286d] for Scintilla bug #1794
+ 0 401   0 | # https://zsh.sourceforge.io/Doc/Release/Expansion.html
+ 1 400   0   
+ 0 400   0   # Where # does not start a comment
+ 1 400   0   
+ 1 400   0   
+ 0 400   0   ## Formatting base
+ 0 400   0   print $(( [#8] y = 33 ))
+ 0 400   0   print $(( [##8] 32767 ))
+ 1 400   0   
+ 0 400   0   # Formatting base and grouping
+ 0 400   0   print $(( [#16_4] 65536 ** 2 ))
+ 1 400   0   
+ 1 400   0   
+ 0 400   0   ## Character values
+ 0 400   0   print $(( ##T+0 ))
+ 0 400   0   print $(( ##^G+0 ))
+ 0 400   0   # Failure: does not work when - included for bindkey syntax. \M-\C-x means Meta+Ctrl+x.
+ 0 400   0   print $(( ##\M-\C-x+0 ))
+ 1 400   0   
+ 0 400   0   # Value of first character of variable in expression
+ 0 400   0   var=Tree
+ 0 400   0   print $(( #var+0 ))
+ 1 400   0   
+ 1 400   0   
+ 0 400   0   ## Extended glob
+ 0 400   0   setopt extended_glob
+ 1 400   0   
+ 0 400   0   # # is similar to *, ## similar to +
+ 0 400   0   echo [A-Za-z]#.bsh
+ 0 400   0   echo [A-Za-z]##.bsh
+ 1 400   0   
+ 0 400   0   # 13 character file names
+ 0 400   0   echo **/[a-zA-Z.](#c13)
+ 0 400   0   # 13-15 character file names
+ 0 400   0   echo **/[a-zA-Z.](#c13,15)
+ 1 400   0   
+ 1 400   0   
+ 0 400   0   ## Glob flag
+ 1 400   0   
+ 0 400   0   # i=case-insensitive
+ 0 400   0   echo (#i)a*
+ 1 400   0   
+ 0 400   0   # b=back-references
+ 0 400   0   foo="a_string_with_a_message"
+ 2 400   0 + if [[ $foo = (a|an)_(#b)(*) ]]; then
+ 0 401   0 |   print ${foo[$mbegin[1],$mend[1]]}
+ 0 401   0 | fi
+ 0 400   0   

lexilla/test/examples/bash/hash.zsh.styled

@@ -0,0 +1,51 @@
+{2}#!/bin/zsh{0}
+{2}# Tests for zsh extensions{0}
+{2}# Can be executed by zsh with reasonable results{0}
+{2}# Some of these were implemented by commit [87286d] for Scintilla bug #1794{0}
+{2}# https://zsh.sourceforge.io/Doc/Release/Expansion.html{0}
+
+{2}# Where # does not start a comment{0}
+
+
+{2}## Formatting base{0}
+{4}print{0} {7}$(({0} {7}[{8}#8{7}]{0} {8}y{0} {7}={0} {3}33{0} {7})){0}
+{4}print{0} {7}$(({0} {7}[{8}##8{7}]{0} {3}32767{0} {7})){0}
+
+{2}# Formatting base and grouping{0}
+{4}print{0} {7}$(({0} {7}[{8}#16_4{7}]{0} {3}65536{0} {7}**{0} {3}2{0} {7})){0}
+
+
+{2}## Character values{0}
+{4}print{0} {7}$(({0} {8}##T{7}+{3}0{0} {7})){0}
+{4}print{0} {7}$(({0} {8}##^G{7}+{3}0{0} {7})){0}
+{2}# Failure: does not work when - included for bindkey syntax. \M-\C-x means Meta+Ctrl+x.{0}
+{4}print{0} {7}$(({0} {8}##\M{7}-{8}\C{7}-{8}x{7}+{3}0{0} {7})){0}
+
+{2}# Value of first character of variable in expression{0}
+{8}var{7}={8}Tree{0}
+{4}print{0} {7}$(({0} {8}#var{7}+{3}0{0} {7})){0}
+
+
+{2}## Extended glob{0}
+{4}setopt{0} {8}extended_glob{0}
+
+{2}# # is similar to *, ## similar to +{0}
+{4}echo{0} {7}[{8}A-Za-z{7}]{8}#.bsh{0}
+{4}echo{0} {7}[{8}A-Za-z{7}]{8}##.bsh{0}
+
+{2}# 13 character file names{0}
+{4}echo{0} {7}**/[{8}a-zA-Z.{7}]{8}(#c13){0}
+{2}# 13-15 character file names{0}
+{4}echo{0} {7}**/[{8}a-zA-Z.{7}]{8}(#c13,15){0}
+
+
+{2}## Glob flag{0}
+
+{2}# i=case-insensitive{0}
+{4}echo{0} {8}(#i)a{7}*{0}
+
+{2}# b=back-references{0}
+{8}foo{7}={5}"a_string_with_a_message"{0}
+{4}if{0} {7}[[{0} {9}$foo{0} {7}={0} {7}({8}a{7}|{8}an{7}){8}_(#b){7}(*){0} {7}]];{0} {4}then{0}
+  {4}print{0} {10}${foo[$mbegin[1],$mend[1]]}{0}
+{4}fi{0}

lexilla/test/examples/bash/x.bsh

@@ -39,4 +39,14 @@ echo $(( x << END ))
 pwd
 END
 
+# Issue 194, failed to understand character escaping so string unterminated
+echo "foo `echo foo \\" bar` bar"
+echo "xinput set-prop bla \"blub\" `grep bla $WRITE_APPENDIX/var/lib/path.txt | cut -d \\" -f 4`" >/some_file.sh
+
+# Issue 194, $ before end of backticks is literal
+echo `echo \$`
+echo `echo \$bar\$`
+echo `echo $`
+echo `echo $bar$`
+
 INVALID_NUMBER=0#0000

lexilla/test/examples/bash/x.bsh.folded

@@ -39,5 +39,15 @@
  0 400   0   pwd
  0 400   0   END
  1 400   0   
+ 0 400   0   # Issue 194, failed to understand character escaping so string unterminated
+ 0 400   0   echo "foo `echo foo \\" bar` bar"
+ 0 400   0   echo "xinput set-prop bla \"blub\" `grep bla $WRITE_APPENDIX/var/lib/path.txt | cut -d \\" -f 4`" >/some_file.sh
+ 1 400   0   
+ 0 400   0   # Issue 194, $ before end of backticks is literal
+ 0 400   0   echo `echo \$`
+ 0 400   0   echo `echo \$bar\$`
+ 0 400   0   echo `echo $`
+ 0 400   0   echo `echo $bar$`
+ 1 400   0   
  0 400   0   INVALID_NUMBER=0#0000
  0 400   0   

lexilla/test/examples/bash/x.bsh.styled

@@ -39,4 +39,14 @@
 {4}pwd{0}
 {8}END{0}
 
+{2}# Issue 194, failed to understand character escaping so string unterminated{0}
+{4}echo{0} {5}"foo `echo foo \\" bar` bar"{0}
+{4}echo{0} {5}"xinput set-prop bla \"blub\" `grep bla $WRITE_APPENDIX/var/lib/path.txt | cut -d \\" -f 4`"{0} {7}>/{8}some_file.sh{0}
+
+{2}# Issue 194, $ before end of backticks is literal{0}
+{4}echo{0} {11}`echo \$`{0}
+{4}echo{0} {11}`echo \$bar\$`{0}
+{4}echo{0} {11}`echo $`{0}
+{4}echo{0} {11}`echo $bar$`{0}
+
 {8}INVALID_NUMBER{7}={1}0#0000{0}

lexilla/test/examples/hypertext/Issue192.html

@@ -0,0 +1,20 @@
+&
+&1
+&A
+&中
+&amp<br />
+&1<br />
+&A<br />
+&中<br />
+&&amp;
+&#1;
+&#1;中
+&A;<br />
+&#1;<br />
+&#1;中<br />
+&amp
+&lt;
+&lt;<br />
+&b.eps;
+&b.eps!
+&—;

lexilla/test/examples/hypertext/Issue192.html.folded

@@ -0,0 +1,21 @@
+ 0 400   0   &
+ 0 400   0   &1
+ 0 400   0   &A
+ 0 400   0   &中
+ 0 400   0   &amp<br />
+ 0 400   0   &1<br />
+ 0 400   0   &A<br />
+ 0 400   0   &中<br />
+ 0 400   0   &&amp;
+ 0 400   0   &#1;
+ 0 400   0   &#1;中
+ 0 400   0   &A;<br />
+ 0 400   0   &#1;<br />
+ 0 400   0   &#1;中<br />
+ 0 400   0   &amp
+ 0 400   0   &lt;
+ 0 400   0   &lt;<br />
+ 0 400   0   &b.eps;
+ 0 400   0   &b.eps!
+ 0 400   0   &—;
+ 0 400   0   

lexilla/test/examples/hypertext/Issue192.html.styled

@@ -0,0 +1,20 @@
+{2}&{0}
+{2}&1{0}
+{2}&A{0}
+{2}&{0}中
+{2}&amp{1}<br{8} {11}/>{0}
+{2}&1{1}<br{8} {11}/>{0}
+{2}&A{1}<br{8} {11}/>{0}
+{2}&{0}中{1}<br{8} {11}/>{0}
+{2}&{10}&amp;{0}
+{10}&#1;{0}
+{10}&#1;{0}中
+{10}&A;{1}<br{8} {11}/>{0}
+{10}&#1;{1}<br{8} {11}/>{0}
+{10}&#1;{0}中{1}<br{8} {11}/>{0}
+{2}&amp{0}
+{10}&lt;{0}
+{10}&lt;{1}<br{8} {11}/>{0}
+{10}&b.eps;{0}
+{2}&b.eps{0}!
+{2}&{0}—;

lexilla/test/examples/hypertext/SciTE.properties

@@ -1,6 +1,6 @@
 lexer.*=hypertext
-keywords.*=b body content head height href html img language link meta \
+keywords.*=b br body content encoding head height href html img language link meta \
-name rel runat script src strong title type width xmlns
+name p rel runat script src strong title type version width xml xmlns
 keywords2.*=function var
 keywords3.*=dim sub
 keywords5.*=echo __file__ __line__

lexilla/test/examples/hypertext/apostophe.php.styled

@@ -8,4 +8,4 @@
 {123}$foo{118} {127}={118} {122}0{124}/*'*/{127};{118}
 {18}?>{0}
 
-{2}<br />{0}
+{1}<br{8} {11}/>{0}

lexilla/test/examples/hypertext/x.html

@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <html xmlns="http://www.w3.org/1999/xhtml">
 <script type="text/javascript">
 var b = /abc/i.test('abc');
@@ -7,6 +8,7 @@ var b = /abc/i.test('abc');
 <head>
     <meta name="Date.Modified" content="20010515" />
     <title>SinkWorld - Portability</title>
+    §
     <unknown>SinkWorld - Portability</unknown>
     <img src="SciTEIco.png" height=64 width=64 />
     <link rel="stylesheet" type="text/css" href="SW.css">

lexilla/test/examples/hypertext/x.html.folded

@@ -1,3 +1,4 @@
+ 0 400   0   <?xml version="1.0" encoding="UTF-8"?>
  2 400   0 + <html xmlns="http://www.w3.org/1999/xhtml">
  2 401   0 + <script type="text/javascript">
  0 402   0 | var b = /abc/i.test('abc');
@@ -7,6 +8,7 @@
  2 401   0 + <head>
  0 402   0 |     <meta name="Date.Modified" content="20010515" />
  0 402   0 |     <title>SinkWorld - Portability</title>
+ 0 402   0 |     §
  0 402   0 |     <unknown>SinkWorld - Portability</unknown>
  0 402   0 |     <img src="SciTEIco.png" height=64 width=64 />
  0 402   0 |     <link rel="stylesheet" type="text/css" href="SW.css">

lexilla/test/examples/hypertext/x.html.styled

@@ -1,3 +1,4 @@
+{12}<?{1}xml{8} {3}version{8}={6}"1.0"{8} {3}encoding{8}={6}"UTF-8"{13}?>{0}
 {1}<html{8} {3}xmlns{8}={6}"http://www.w3.org/1999/xhtml"{1}>{0}
 {1}<script{8} {3}type{8}={6}"text/javascript"{1}>{40}
 {47}var{41} {46}b{41} {50}={41} {52}/abc/i{46}.test{50}({49}'abc'{50});{41}
@@ -7,6 +8,7 @@
 {1}<head>{0}
     {1}<meta{8} {3}name{8}={6}"Date.Modified"{8} {3}content{8}={6}"20010515"{8} {11}/>{0}
     {1}<title>{0}SinkWorld - Portability{1}</title>{0}
+    §
     {2}<unknown>{0}SinkWorld - Portability{2}</unknown>{0}
     {1}<img{8} {3}src{8}={6}"SciTEIco.png"{8} {3}height{8}={5}64{8} {3}width{8}={5}64{8} {11}/>{0}
     {1}<link{8} {3}rel{8}={6}"stylesheet"{8} {3}type{8}={6}"text/css"{8} {3}href{8}={6}"SW.css"{1}>{0}

lexilla/test/examples/verilog/AllStyles.vh

@@ -0,0 +1,97 @@
+// Examples drawn from https://verilogams.com/refman/basics/index.html
+
+// SCE_V_DEFAULT {0}
+
+/*
+ * SCE_V_COMMENT {1}
+ */
+
+// SCE_V_COMMENTLINE {2}
+// multiple
+// comment lines
+// are folded
+
+//{ explicit folds
+//  are folded,
+//} too
+
+//! SCE_V_COMMENTLINEBANG {3}
+//! multiple
+//! bang comments
+//! are folded
+
+// SCE_V_NUMBER {4}
+1'b0
+8'hx
+8'hfffx
+12'hfx
+64'o0
+0x7f
+0o23
+0b1011
+42_839
+0.1
+1.3u
+5.46K
+1.2E12
+1.30e-2
+236.123_763e-12
+
+// SCE_V_WORD {5}
+always
+
+// SCE_V_STRING {6}
+"\tsome\ttext\r\n"
+
+// SCE_V_WORD2 {7}
+special
+
+// SCE_V_WORD3 {8}
+$async$and$array
+
+// SCE_V_PREPROCESSOR {9}
+`define __VAMS_ENABLE__
+`ifdef __VAMS_ENABLE__
+    parameter integer del = 1 from [1:100];
+`else
+    parameter del = 1;
+`endif
+
+// SCE_V_OPERATOR {10}
++-/=!@#%^&*()[]{}<|>~
+
+// SCE_V_IDENTIFIER {11}
+q
+x$z
+\my_var
+\/x1/n1
+\\x1\n1
+\{a,b}
+\V(p,n)
+
+// SCE_V_STRINGEOL {12}
+"\n
+
+// SCE_V_USER {19}
+my_var
+
+// SCE_V_COMMENT_WORD {20}
+// TODO write a comment
+
+module mod(clk, q, reset) // folded when fold.verilog.flags=1
+// SCE_V_INPUT {21}
+  input clk;
+
+// SCE_V_OUTPUT {22}
+  output q;
+
+// SCE_V_INOUT {23}
+  inout reset;
+endmodule
+
+// SCE_V_PORT_CONNECT {24}
+mod m1(
+  .clk(clk),
+  .q(q),
+  .reset(reset)
+);

lexilla/test/examples/verilog/AllStyles.vh.folded

@@ -0,0 +1,98 @@
+ 0 400 400   // Examples drawn from https://verilogams.com/refman/basics/index.html
+ 0 400 400   
+ 0 400 400   // SCE_V_DEFAULT {0}
+ 0 400 400   
+ 2 400 401 + /*
+ 0 401 401 |  * SCE_V_COMMENT {1}
+ 0 401 400 |  */
+ 0 400 400   
+ 2 400 401 + // SCE_V_COMMENTLINE {2}
+ 0 401 401 | // multiple
+ 0 401 401 | // comment lines
+ 0 401 400 | // are folded
+ 0 400 400   
+ 2 400 402 + //{ explicit folds
+ 0 402 402 | //  are folded,
+ 0 402 400 | //} too
+ 0 400 400   
+ 2 400 401 + //! SCE_V_COMMENTLINEBANG {3}
+ 0 401 401 | //! multiple
+ 0 401 401 | //! bang comments
+ 0 401 400 | //! are folded
+ 0 400 400   
+ 0 400 400   // SCE_V_NUMBER {4}
+ 0 400 400   1'b0
+ 0 400 400   8'hx
+ 0 400 400   8'hfffx
+ 0 400 400   12'hfx
+ 0 400 400   64'o0
+ 0 400 400   0x7f
+ 0 400 400   0o23
+ 0 400 400   0b1011
+ 0 400 400   42_839
+ 0 400 400   0.1
+ 0 400 400   1.3u
+ 0 400 400   5.46K
+ 0 400 400   1.2E12
+ 0 400 400   1.30e-2
+ 0 400 400   236.123_763e-12
+ 0 400 400   
+ 0 400 400   // SCE_V_WORD {5}
+ 0 400 400   always
+ 0 400 400   
+ 0 400 400   // SCE_V_STRING {6}
+ 0 400 400   "\tsome\ttext\r\n"
+ 0 400 400   
+ 0 400 400   // SCE_V_WORD2 {7}
+ 0 400 400   special
+ 0 400 400   
+ 0 400 400   // SCE_V_WORD3 {8}
+ 0 400 400   $async$and$array
+ 0 400 400   
+ 0 400 400   // SCE_V_PREPROCESSOR {9}
+ 0 400 400   `define __VAMS_ENABLE__
+ 2 400 401 + `ifdef __VAMS_ENABLE__
+ 0 401 401 |     parameter integer del = 1 from [1:100];
+ 2 400 401 + `else
+ 0 401 401 |     parameter del = 1;
+ 0 401 400 | `endif
+ 0 400 400   
+ 0 400 400   // SCE_V_OPERATOR {10}
+ 0 400 400   +-/=!@#%^&*()[]{}<|>~
+ 0 400 400   
+ 0 400 400   // SCE_V_IDENTIFIER {11}
+ 0 400 400   q
+ 0 400 400   x$z
+ 0 400 400   \my_var
+ 0 400 400   \/x1/n1
+ 0 400 400   \\x1\n1
+ 0 400 400   \{a,b}
+ 0 400 400   \V(p,n)
+ 0 400 400   
+ 0 400 400   // SCE_V_STRINGEOL {12}
+ 0 400 400   "\n
+ 0 400 400   
+ 0 400 400   // SCE_V_USER {19}
+ 0 400 400   my_var
+ 0 400 400   
+ 2 400 401 + // SCE_V_COMMENT_WORD {20}
+ 0 401 400 | // TODO write a comment
+ 0 400 400   
+ 2 400 401 + module mod(clk, q, reset) // folded when fold.verilog.flags=1
+ 0 401 401 | // SCE_V_INPUT {21}
+ 0 401 401 |   input clk;
+ 0 401 401 | 
+ 0 401 401 | // SCE_V_OUTPUT {22}
+ 0 401 401 |   output q;
+ 0 401 401 | 
+ 0 401 401 | // SCE_V_INOUT {23}
+ 0 401 401 |   inout reset;
+ 0 401 400 | endmodule
+ 0 400 400   
+ 0 400 400   // SCE_V_PORT_CONNECT {24}
+ 2 400 401 + mod m1(
+ 0 401 401 |   .clk(clk),
+ 0 401 401 |   .q(q),
+ 0 401 401 |   .reset(reset)
+ 0 401 400 | );
+ 0 400   0   

lexilla/test/examples/verilog/AllStyles.vh.styled

@@ -0,0 +1,97 @@
+{2}// Examples drawn from https://verilogams.com/refman/basics/index.html
+{0}
+{2}// SCE_V_DEFAULT {0}
+{0}
+{1}/*
+ * SCE_V_COMMENT {1}
+ */{0}
+
+{2}// SCE_V_COMMENTLINE {2}
+// multiple
+// comment lines
+// are folded
+{0}
+{2}//{ explicit folds
+//  are folded,
+//} too
+{0}
+{3}//! SCE_V_COMMENTLINEBANG {3}
+//! multiple
+//! bang comments
+//! are folded
+{0}
+{2}// SCE_V_NUMBER {4}
+{4}1'b0{0}
+{4}8'hx{0}
+{4}8'hfffx{0}
+{4}12'hfx{0}
+{4}64'o0{0}
+{4}0x7f{0}
+{4}0o23{0}
+{4}0b1011{0}
+{4}42_839{0}
+{4}0.1{0}
+{4}1.3u{0}
+{4}5.46K{0}
+{4}1.2E12{0}
+{4}1.30e{10}-{4}2{0}
+{4}236.123_763e{10}-{4}12{0}
+
+{2}// SCE_V_WORD {5}
+{5}always{0}
+
+{2}// SCE_V_STRING {6}
+{6}"\tsome\ttext\r\n"{0}
+
+{2}// SCE_V_WORD2 {7}
+{7}special{0}
+
+{2}// SCE_V_WORD3 {8}
+{8}$async$and$array{0}
+
+{2}// SCE_V_PREPROCESSOR {9}
+{9}`define{0} {11}__VAMS_ENABLE__{0}
+{9}`ifdef{0} {11}__VAMS_ENABLE__{0}
+    {5}parameter{0} {5}integer{0} {11}del{0} {10}={0} {4}1{0} {11}from{0} {10}[{4}1{10}:{4}100{10}];{0}
+{9}`else{64}
+    {69}parameter{64} {75}del{64} {74}={64} {68}1{74};{64}
+{9}`endif{0}
+
+{2}// SCE_V_OPERATOR {10}
+{10}+-/=!@#%^&*()[]{}<|>~{0}
+
+{2}// SCE_V_IDENTIFIER {11}
+{11}q{0}
+{11}x$z{0}
+{11}\my_var{0}
+{11}\/x1/n1{0}
+{11}\\x1\n1{0}
+{11}\{a,b}{0}
+{11}\V(p,n){0}
+
+{2}// SCE_V_STRINGEOL {12}
+{12}"\n
+{0}
+{2}// SCE_V_USER {19}
+{19}my_var{0}
+
+{2}// SCE_V_COMMENT_WORD {20}
+// {20}TODO{2} write a comment
+{0}
+{5}module{0} {11}mod{10}({11}clk{10},{0} {11}q{10},{0} {11}reset{10}){0} {2}// folded when fold.verilog.flags=1
+// SCE_V_INPUT {21}
+{0}  {21}input{0} {21}clk{10};{0}
+
+{2}// SCE_V_OUTPUT {22}
+{0}  {22}output{0} {22}q{10};{0}
+
+{2}// SCE_V_INOUT {23}
+{0}  {23}inout{0} {23}reset{10};{0}
+{5}endmodule{0}
+
+{2}// SCE_V_PORT_CONNECT {24}
+{11}mod{0} {11}m1{10}({0}
+  {10}.{24}clk{10}({11}clk{10}),{0}
+  {10}.{24}q{10}({11}q{10}),{0}
+  {10}.{24}reset{10}({11}reset{10}){0}
+{10});{0}

lexilla/test/examples/verilog/SciTE.properties

@@ -0,0 +1,62 @@
+lexer.*.vh=verilog
+
+keywords.*.vh= \
+always and assign automatic \
+begin buf bufif0 bufif1 \
+case casex casez cell cmos config \
+deassign default defparam design disable \
+edge else end endcase endconfig endfunction endgenerate endmodule endprimitive endspecify endtable endtask event \
+for force forever fork function \
+generate genvar \
+highz0 highz1 \
+if ifnone incdir include initial inout input instance integer \
+join \
+large liblist library localparam \
+macromodule medium module \
+nand negedge nmos nor noshowcancelled not notif0 notif1 \
+or output \
+parameter pmos posedge primitive pull0 pull1 pulldown pullup pulsestyle_ondetect pulsestyle_onevent \
+rcmos real realtime reg release repeat rnmos rpmos rtran rtranif0 rtranif1 \
+scalared showcancelled signed small specify specparam strong0 strong1 supply0 supply1 \
+table task time tran tranif0 tranif1 tri tri0 tri1 triand trior trireg \
+unsigned use uwire \
+vectored \
+wait wand weak0 weak1 while wire wor \
+xnor xor
+keywords2.*.vh=special
+keywords3.*.vh= \
+$async$and$array $async$and$plane $async$nand$array $async$nand$plane $async$nor$array $async$nor$plane $async$or$array $async$or$plane \
+$bitstoreal \
+$countdrivers \
+$display $displayb $displayh $displayo \
+$dist_chi_square $dist_erlang $dist_exponential $dist_normal $dist_poisson $dist_t $dist_uniform \
+$dumpall $dumpfile $dumpflush $dumplimit $dumpoff $dumpon $dumpportsall $dumpportsflush $dumpportslimit $dumpportsoff $dumpportson $dumpvars \
+$fclose $fdisplayh $fdisplay $fdisplayf $fdisplayb $feof $ferror $fflush $fgetc $fgets $finish $fmonitorb $fmonitor $fmonitorf $fmonitorh $fopen $fread $fscanf $fseek $fsscanf $fstrobe $fstrobebb $fstrobef $fstrobeh $ftel $fullskew $fwriteb $fwritef $fwriteh $fwrite \
+$getpattern \
+$history $hold \
+$incsave $input $itor \
+$key \
+$list $log \
+$monitorb $monitorh $monitoroff $monitoron $monitor $monitoro \
+$nochange $nokey $nolog \
+$period $printtimescale \
+$q_add $q_exam $q_full $q_initialize $q_remove \
+$random $readmemb $readmemh $readmemh $realtime $realtobits $recovery $recrem $removal $reset_count $reset $reset_value $restart $rewind $rtoi \
+$save $scale $scope $sdf_annotate $setup $setuphold $sformat $showscopes $showvariables $showvars $signed $skew $sreadmemb $sreadmemh $stime $stop $strobeb $strobe $strobeh $strobeo $swriteb $swriteh $swriteo $swrite $sync$and$array $sync$and$plane $sync$nand$array $sync$nand$plane $sync$nor$array $sync$nor$plane $sync$or$array $sync$or$plane \
+$test$plusargs $time $timeformat $timeskew \
+$ungetc $unsigned \
+$value$plusargs \
+$width $writeb $writeh $write $writeo
+keywords4.*.vh=my_var
+keywords5.*.vh=synopsys parallel_case infer_mux TODO
+
+fold=1
+fold.compact=0
+fold.comment=1
+fold.preprocessor=1
+fold.at.else=1
+fold.verilog.flags=1  # fold module definitions
+lexer.verilog.track.preprocessor=1
+lexer.verilog.update.preprocessor=1
+lexer.verilog.portstyling=1
+lexer.verilog.fold.preprocessor.else=1

lexilla/version.txt

@@ -1 +1 @@
-526
+527

scintilla/cocoa/Scintilla/Info.plist

@@ -15,7 +15,7 @@
 	<key>CFBundlePackageType</key>
 	<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
 	<key>CFBundleShortVersionString</key>
-	<string>5.3.6</string>
+	<string>5.3.7</string>
 	<key>CFBundleVersion</key>
 	<string>$(CURRENT_PROJECT_VERSION)</string>
 	<key>NSHumanReadableCopyright</key>

scintilla/cocoa/Scintilla/Scintilla.xcodeproj/project.pbxproj

@@ -573,7 +573,7 @@
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 5.3.6;
+				CURRENT_PROJECT_VERSION = 5.3.7;
 				DEAD_CODE_STRIPPING = YES;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
@@ -636,7 +636,7 @@
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
 				COPY_PHASE_STRIP = NO;
-				CURRENT_PROJECT_VERSION = 5.3.6;
+				CURRENT_PROJECT_VERSION = 5.3.7;
 				DEAD_CODE_STRIPPING = YES;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				ENABLE_NS_ASSERTIONS = NO;
@@ -667,7 +667,7 @@
 				CODE_SIGN_IDENTITY = "-";
 				CODE_SIGN_STYLE = Manual;
 				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 5.3.6;
+				CURRENT_PROJECT_VERSION = 5.3.7;
 				DEAD_CODE_STRIPPING = YES;
 				DEFINES_MODULE = YES;
 				DEVELOPMENT_TEAM = "";
@@ -702,7 +702,7 @@
 				CODE_SIGN_IDENTITY = "-";
 				CODE_SIGN_STYLE = Manual;
 				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 5.3.6;
+				CURRENT_PROJECT_VERSION = 5.3.7;
 				DEAD_CODE_STRIPPING = YES;
 				DEFINES_MODULE = YES;
 				DEVELOPMENT_TEAM = "";

scintilla/doc/ScintillaDownload.html

@@ -26,9 +26,9 @@
     <table bgcolor="#CCCCCC" width="100%" cellspacing="0" cellpadding="8" border="0">
       <tr>
         <td>
-          <font size="4"> <a href="https://www.scintilla.org/scintilla536.zip">
+          <font size="4"> <a href="https://www.scintilla.org/scintilla537.zip">
 	Windows</a>&nbsp;&nbsp;
-	<a href="https://www.scintilla.org/scintilla536.tgz">
+	<a href="https://www.scintilla.org/scintilla537.tgz">
           GTK/Linux</a>&nbsp;&nbsp;
 	</font>
         </td>
@@ -42,7 +42,7 @@
       containing very few restrictions.
     </p>
     <h3>
-       Release 5.3.6
+       Release 5.3.7
     </h3>
     <h4>
        Source Code
@@ -50,8 +50,8 @@
        The source code package contains all of the source code for Scintilla but no binary
 	executable code and is available in
        <ul>
-       <li><a href="https://www.scintilla.org/scintilla536.zip">zip format</a> (1.4M) commonly used on Windows</li>
+       <li><a href="https://www.scintilla.org/scintilla537.zip">zip format</a> (1.7M) commonly used on Windows</li>
-       <li><a href="https://www.scintilla.org/scintilla536.tgz">tgz format</a> (1.3M) commonly used on Linux and compatible operating systems</li>
+       <li><a href="https://www.scintilla.org/scintilla537.tgz">tgz format</a> (1.6M) commonly used on Linux and compatible operating systems</li>
        </ul>
        Instructions for building on both Windows and Linux are included in the readme file.
     <h4>

scintilla/doc/ScintillaHistory.html

@@ -578,9 +578,38 @@
 	<td>Enrico Tröger</td>
       </tr><tr>
 	<td>Chengzhi Li</td>
+	<td>Gary James</td>
+	<td>Tsuyoshi Miyake</td>
     </tr>
     </table>
     <h2>Releases</h2>
+    <h3>
+       <a href="https://www.scintilla.org/scintilla537.zip">Release 5.3.7</a>
+    </h3>
+    <ul>
+	<li>
+	Released 22 September 2023.
+	</li>
+	<li>
+	For GTK on macOS, fix popup window behaviour by setting type hints.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2401/">Bug #2401</a>.
+	</li>
+	<li>
+	For GTK, fix assertion failure on some systems when an INDIC_SQUIGGLEPIXMAP drawn
+	for a zero-width character.
+	</li>
+	<li>
+	For Qt, allow parent window to handle context menu events by setting as ignored.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2395/">Bug #2395</a>.
+	</li>
+	<li>
+	For Qt, fix potential crash when using IME with large amount of text selected.
+	</li>
+	<li>
+	For Windows, fix building with non-English environment.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2400/">Bug #2400</a>.
+	</li>
+    </ul>
     <h3>
        <a href="https://www.scintilla.org/scintilla536.zip">Release 5.3.6</a>
     </h3>

scintilla/doc/index.html

@@ -9,7 +9,7 @@
     <meta name="keywords" content="Scintilla, SciTE, Editing Component, Text Editor" />
     <meta name="Description"
     content="www.scintilla.org is the home of the Scintilla editing component and SciTE text editor application." />
-    <meta name="Date.Modified" content="20230726" />
+    <meta name="Date.Modified" content="20230922" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <style type="text/css">
         .logo {
@@ -60,8 +60,8 @@
           GTK, and macOS</font>
         </td>
         <td width="40%" align="right">
-          <font color="#FFCC99" size="3"> Release version 5.3.6<br />
+          <font color="#FFCC99" size="3"> Release version 5.3.7<br />
-           Site last modified July 26 2023</font>
+           Site last modified September 22 2023</font>
         </td>
         <td width="20%">
           &nbsp;
@@ -76,11 +76,11 @@
       </tr>
     </table>
     <ul id="versionlist">
+      <li>Version 5.3.7 fixes platform-specific issues on GTK, Qt, and Win32.</li>
       <li>Version 5.3.6 improves cursor behaviour on Win32 and IME support on Win32 and Qt.</li>
       <li>Version 5.3.5 improves IME support on Win32 and Qt.</li>
       <li>Version 5.3.4 adds multithreaded wrapping.</li>
       <li>Version 5.3.3 fixes minor bugs in APIs and platform layers.</li>
-      <li>Version 5.3.2 adds SCI_REPLACETARGETMINIMAL to modify text without marking unchanged start and end text in change history.</li>
     </ul>
     <ul id="menu">
       <li id="remote1"><a href="https://www.scintilla.org/SciTEImage.html">Screenshot</a></li>

scintilla/gtk/PlatGTK.cxx

@@ -607,6 +607,8 @@ void SurfaceImpl::GradientRectangle(PRectangle rc, const std::vector<ColourStop>
 
 void SurfaceImpl::DrawRGBAImage(PRectangle rc, int width, int height, const unsigned char *pixelsImage) {
 	PLATFORM_ASSERT(context);
+	if (width == 0)
+		return;
 	if (rc.Width() > width)
 		rc.left += (rc.Width() - width) / 2;
 	rc.right = rc.left + width;
@@ -1622,6 +1624,7 @@ void ListBoxX::Create(Window &parent, int, Point, int, bool, Technology) {
 #endif
 
 	wid = widCached = gtk_window_new(GTK_WINDOW_POPUP);
+	gtk_window_set_type_hint(GTK_WINDOW(wid), GDK_WINDOW_TYPE_HINT_POPUP_MENU);
 
 	frame = gtk_frame_new(nullptr);
 	gtk_widget_show(PWidget(frame));

scintilla/gtk/ScintillaGTK.cxx

@@ -673,6 +673,7 @@ void ScintillaGTK::Init() {
 
 	/* create pre-edit window */
 	wPreedit = gtk_window_new(GTK_WINDOW_POPUP);
+	gtk_window_set_type_hint(GTK_WINDOW(PWidget(wPreedit)), GDK_WINDOW_TYPE_HINT_POPUP_MENU);
 	wPreeditDraw = gtk_drawing_area_new();
 	GtkWidget *predrw = PWidget(wPreeditDraw);      // No code inside the G_OBJECT macro
 #if GTK_CHECK_VERSION(3,0,0)
@@ -1406,6 +1407,7 @@ void ScintillaGTK::Paste() {
 void ScintillaGTK::CreateCallTipWindow(PRectangle rc) {
 	if (!ct.wCallTip.Created()) {
 		ct.wCallTip = gtk_window_new(GTK_WINDOW_POPUP);
+		gtk_window_set_type_hint(GTK_WINDOW(PWidget(ct.wCallTip)), GDK_WINDOW_TYPE_HINT_TOOLTIP);
 		ct.wDraw = gtk_drawing_area_new();
 		GtkWidget *widcdrw = PWidget(ct.wDraw);	//	// No code inside the G_OBJECT macro
 		gtk_container_add(GTK_CONTAINER(PWidget(ct.wCallTip)), widcdrw);

scintilla/qt/ScintillaEdit/ScintillaEdit.pro

@@ -13,7 +13,7 @@ TEMPLATE = lib
 CONFIG += lib_bundle
 CONFIG += c++1z
 
-VERSION = 5.3.6
+VERSION = 5.3.7
 
 SOURCES += \
     ScintillaEdit.cpp \

scintilla/qt/ScintillaEditBase/ScintillaEditBase.cpp

@@ -372,6 +372,9 @@ void ScintillaEditBase::contextMenuEvent(QContextMenuEvent *event)
 	}
 	if (sqt->ShouldDisplayPopup(pt)) {
 		sqt->ContextMenu(pos);
+		event->accept();
+	} else {
+		event->ignore();
 	}
 }
 
@@ -671,7 +674,7 @@ QVariant ScintillaEditBase::inputMethodQuery(Qt::InputMethodQuery query) const
 
 		case Qt::ImCurrentSelection:
 		{
-			QVarLengthArray<char,1024> buffer(send(SCI_GETSELTEXT));
+			QVarLengthArray<char,1024> buffer(send(SCI_GETSELTEXT)+1);
 			sends(SCI_GETSELTEXT, 0, buffer.data());
 
 			return sqt->StringFromDocument(buffer.constData());

scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro

@@ -12,7 +12,7 @@ TEMPLATE = lib
 CONFIG += lib_bundle
 CONFIG += c++1z
 
-VERSION = 5.3.6
+VERSION = 5.3.7
 
 SOURCES += \
     PlatQt.cpp \

scintilla/version.txt

@@ -1 +1 @@
-536
+537

scintilla/win32/ScintRes.rc

@@ -4,8 +4,8 @@
 
 #include <windows.h>
 
-#define VERSION_SCINTILLA "5.3.6"
+#define VERSION_SCINTILLA "5.3.7"
-#define VERSION_WORDS 5, 3, 6, 0
+#define VERSION_WORDS 5, 3, 7, 0
 
 VS_VERSION_INFO VERSIONINFO
 FILEVERSION	VERSION_WORDS

scintilla/win32/Scintilla.vcxproj

@@ -97,6 +97,7 @@
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <MinimalRebuild>false</MinimalRebuild>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <AdditionalOptions>/source-charset:utf-8 %(AdditionalOptions)</AdditionalOptions>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>

scintilla/win32/scintilla.mak

@@ -36,7 +36,7 @@ SUBSYSTEM=-SUBSYSTEM:WINDOWS,10.00
 !ENDIF
 
 CRTFLAGS=-D_CRT_SECURE_NO_DEPRECATE=1 $(ADD_DEFINE)
-CXXFLAGS=-Zi -TP -MP -W4 -EHsc -std:c++17 $(CRTFLAGS)
+CXXFLAGS=-Zi -TP -MP -W4 -EHsc -std:c++17 -utf-8 $(CRTFLAGS)
 CXXDEBUG=-Od -MTd -DDEBUG
 CXXNDEBUG=-O2 -MT -DNDEBUG -GL
 NAME=-Fo