Add ability to remove any duplicate lines in a document, keeping the first occurrence

Fix #8965, close #9033

PowerEditor/installer/nativeLang/english.xml

@@ -112,6 +112,7 @@ The comments are here for explanation, it's not necessary to translate them.
                     <Item id="42008" name="Increase Line Indent"/>
                     <Item id="42009" name="Decrease Line Indent"/>
                     <Item id="42010" name="Duplicate Current Line"/>
+                    <Item id="42079" name="Remove Duplicate Lines"/>
                     <Item id="42077" name="Remove Consecutive Duplicate Lines"/>
                     <Item id="42012" name="Split Lines"/>
                     <Item id="42013" name="Join Lines"/>

PowerEditor/src/MISC/Common/Common.h

@@ -31,6 +31,8 @@
 #include <windows.h>
 #include <iso646.h>
 #include <cstdint>
+#include <unordered_set>
+#include <algorithm>
 
 
 const bool dirUp = true;
@@ -205,3 +207,29 @@ std::string ws2s(const std::wstring& wstr);
 bool deleteFileOrFolder(const generic_string& f2delete);
 
 void getFilesInFolder(std::vector<generic_string>& files, const generic_string& extTypeFilter, const generic_string& inFolder);
+
+template<typename T> size_t vecRemoveDuplicates(std::vector<T>& vec, bool isSorted = false, bool canSort = false)
+{
+	if (!isSorted && canSort)
+	{
+		std::sort(vec.begin(), vec.end());
+		isSorted = true;
+	}
+
+	if (isSorted)
+	{
+		typename std::vector<T>::iterator it;
+		it = std::unique(vec.begin(), vec.end());
+		vec.resize(distance(vec.begin(), it));  // unique() does not shrink the vector
+	}
+	else
+	{
+		std::unordered_set<T> seen;
+		auto newEnd = std::remove_if(vec.begin(), vec.end(), [&seen](const T& value)
+			{
+				return !seen.insert(value).second;
+			});
+		vec.erase(newEnd, vec.end());
+	}
+	return vec.size();
+}

PowerEditor/src/Notepad_plus.rc

@@ -308,6 +308,7 @@ BEGIN
         POPUP "Line Operations"
         BEGIN
             MENUITEM "Duplicate Current Line",                           IDM_EDIT_DUP_LINE
+            MENUITEM "Remove Duplicate Lines",                           IDM_EDIT_REMOVE_ANY_DUP_LINES
             MENUITEM "Remove Consecutive Duplicate Lines",               IDM_EDIT_REMOVE_DUP_LINES
             MENUITEM "Split Lines",                                      IDM_EDIT_SPLIT_LINES
             MENUITEM "Join Lines",                                       IDM_EDIT_JOIN_LINES

PowerEditor/src/NppCommands.cpp

@@ -1546,6 +1546,12 @@ void Notepad_plus::command(int id)
 			_pEditView->execute(SCI_ENDUNDOACTION);
 			break;
 
+		case IDM_EDIT_REMOVE_ANY_DUP_LINES:
+			_pEditView->execute(SCI_BEGINUNDOACTION);
+			_pEditView->removeAnyDuplicateLines();
+			_pEditView->execute(SCI_ENDUNDOACTION);
+			break;
+
 		case IDM_EDIT_SPLIT_LINES:
 		{
 			if (_pEditView->execute(SCI_GETSELECTIONS) == 1)
@@ -3524,6 +3530,7 @@ void Notepad_plus::command(int id)
 			case IDM_EDIT_RMV_TAB:
 			case IDM_EDIT_DUP_LINE:
 			case IDM_EDIT_REMOVE_DUP_LINES:
+			case IDM_EDIT_REMOVE_ANY_DUP_LINES:
 			case IDM_EDIT_TRANSPOSE_LINE:
 			case IDM_EDIT_SPLIT_LINES:
 			case IDM_EDIT_JOIN_LINES:

PowerEditor/src/Parameters.cpp

@@ -128,6 +128,7 @@ static const WinMenuKeyDefinition winKeyDefs[] =
 	{ VK_NULL,    IDM_EDIT_INVERTCASE,                          false, false, false, nullptr },
 	{ VK_NULL,    IDM_EDIT_RANDOMCASE,                          false, false, false, nullptr },
 	{ VK_NULL,    IDM_EDIT_REMOVE_DUP_LINES,                    false, false, false, nullptr },
+	{ VK_NULL,    IDM_EDIT_REMOVE_ANY_DUP_LINES,                false, false, false, nullptr },
 	{ VK_I,       IDM_EDIT_SPLIT_LINES,                         true,  false, false, nullptr },
 	{ VK_J,       IDM_EDIT_JOIN_LINES,                          true,  false, false, nullptr },
 	{ VK_UP,      IDM_EDIT_LINE_UP,                             true,  false, true,  nullptr },

PowerEditor/src/ScitillaComponent/ScintillaEditView.cpp

@@ -3871,3 +3871,66 @@ void ScintillaEditView::markedTextToClipboard(int indiStyle, bool doAll /*= fals
 		str2Clipboard(joined, NULL);
 	}
 }
+
+void ScintillaEditView::removeAnyDuplicateLines()
+{
+	size_t fromLine = 0, toLine = 0;
+	bool hasLineSelection = false;
+
+	auto selStart = execute(SCI_GETSELECTIONSTART);
+	auto selEnd = execute(SCI_GETSELECTIONEND);
+	hasLineSelection = selStart != selEnd;
+
+	if (hasLineSelection)
+	{
+		const pair<int, int> lineRange = getSelectionLinesRange();
+		// One single line selection is not allowed.
+		if (lineRange.first == lineRange.second)
+		{
+			return;
+		}
+		fromLine = lineRange.first;
+		toLine = lineRange.second;
+	}
+	else
+	{
+		// No selection.
+		fromLine = 0;
+		toLine = execute(SCI_GETLINECOUNT) - 1;
+	}
+
+	if (fromLine >= toLine)
+	{
+		return;
+	}
+
+	const auto startPos = execute(SCI_POSITIONFROMLINE, fromLine);
+	const auto endPos = execute(SCI_POSITIONFROMLINE, toLine) + execute(SCI_LINELENGTH, toLine);
+	const generic_string text = getGenericTextAsString(startPos, endPos);
+	std::vector<generic_string> linesVect = stringSplit(text, getEOLString());
+	const size_t lineCount = execute(SCI_GETLINECOUNT);
+
+	const bool doingEntireDocument = toLine == lineCount - 1;
+	if (!doingEntireDocument)
+	{
+		if (linesVect.rbegin()->empty())
+		{
+			linesVect.pop_back();
+		}
+	}
+
+	size_t origSize = linesVect.size();
+	size_t newSize = vecRemoveDuplicates(linesVect);
+	if (origSize != newSize)
+	{
+		generic_string joined = stringJoin(linesVect, getEOLString());
+		if (!doingEntireDocument)
+		{
+			joined += getEOLString();
+		}
+		if (text != joined)
+		{
+			replaceTarget(joined.c_str(), int(startPos), int(endPos));
+		}
+	}
+}

PowerEditor/src/ScitillaComponent/ScintillaEditView.h

@@ -621,6 +621,7 @@ public:
 	bool isTextDirectionRTL() const;
 	void setPositionRestoreNeeded(bool val) { _positionRestoreNeeded = val; };
 	void markedTextToClipboard(int indiStyle, bool doAll = false);
+	void removeAnyDuplicateLines();
 
 protected:
 	static HINSTANCE _hLib;

PowerEditor/src/menuCmdID.h

@@ -106,6 +106,7 @@
     #define    IDM_EDIT_RMV_TAB                     (IDM_EDIT + 9)
     #define    IDM_EDIT_DUP_LINE                    (IDM_EDIT + 10)
     #define    IDM_EDIT_REMOVE_DUP_LINES            (IDM_EDIT + 77)
+    #define    IDM_EDIT_REMOVE_ANY_DUP_LINES        (IDM_EDIT + 79)
     #define    IDM_EDIT_TRANSPOSE_LINE              (IDM_EDIT + 11)
     #define    IDM_EDIT_SPLIT_LINES                 (IDM_EDIT + 12)
     #define    IDM_EDIT_JOIN_LINES                  (IDM_EDIT + 13)