Results of findDupLines.js from Macros & Scripts downloads page

Peter · Apr 20, 2018#12018-04-20T10:43+00:00

I downloaded the script file findDupLines.js from UltraEdit macros and scripts download page.

Why are lines 13 and 14 displayed as "Duplicate"?

Mofi · Apr 21, 2018#22018-04-21T09:09+00:00

Well, script findDupLines.js is indeed simple as written in first line by the anonymous user who contributed this script and has some serious issues.

Control of case sensitivity with variable caseSens does not work at all.
The script does not really check if an entire line is a duplicate of another entire line. It just searches if string of current line is found anywhere below in another line in file.
The script does not search for multiple duplicates of a line and instead reports multiple times duplicates, for example line 3 and 5 are duplicate and line 5 and 8 are duplicate on lines 3, 5 and 8 are partly or completely identical.
The output of the script is not good for evaluating the results.

I wanted to quickly improve this script and fix the issues. But final script is rewritten more or less completely.

Code: Select all

// This is a complex script which searches for duplicate lines.

// THE DUPLICATE CHECK IS CASE SENSITIVE!

// Change the following variable to 0 to disable case sensitivity.

var nCaseSensitive = 1;

if (UltraEdit.document.length > 0)
{
   UltraEdit.insertMode();
   if (typeof(UltraEdit.columnModeOff) == "function") UltraEdit.columnModeOff();
   else if (typeof(UltraEdit.activeDocument.columnModeOff) == "function") UltraEdit.activeDocument.columnModeOff();
   UltraEdit.activeDocument.top();

   UltraEdit.ueReOn();
   UltraEdit.activeDocument.findReplace.mode=0;
   UltraEdit.activeDocument.findReplace.matchCase=false;
   UltraEdit.activeDocument.findReplace.matchWord=false;
   UltraEdit.activeDocument.findReplace.regExp=false;
   UltraEdit.activeDocument.findReplace.searchDown=true;
   if (typeof(UltraEdit.activeDocument.findReplace.searchInColumn) == "boolean")
   {
      UltraEdit.activeDocument.findReplace.searchInColumn=false;
   }

   UltraEdit.outputWindow.write("Beginning duplicate check...");

   if (nCaseSensitive)
   {
      UltraEdit.activeDocument.findReplace.matchCase = true;
      UltraEdit.outputWindow.write("Case sensitivity enabled");
   }
   else
   {
      UltraEdit.activeDocument.findReplace.matchCase = false;
      UltraEdit.outputWindow.write("Case sensitivity disabled");
   }

   var nDuplicates = 0;
   var nLineNumber = 1;
   var nStartIndex = 0;
   var anDuplicateLines = [];
   var sFullFileName = UltraEdit.activeDocument.path;

   while (!UltraEdit.activeDocument.isEof())
   {
      var nIndex = nStartIndex;
      while(nIndex < anDuplicateLines.length)
      {
         if (anDuplicateLines[nIndex] > nLineNumber) break;
         if (nLineNumber == anDuplicateLines[nIndex])
         {
            nStartIndex = nIndex + 1;
            nIndex = -1;
            break;
         }
         nIndex++;
      }
      if (nIndex >= 0)
      {
         UltraEdit.activeDocument.selectLine();
         var sCurrentLine = UltraEdit.activeDocument.selection.replace(/[\r\n]+/g,"");

         if (sCurrentLine.length)
         {
            var bFirstDuplicate = true;
            var sFindLine = sCurrentLine.replace(/\^/g,"^^");
            while (UltraEdit.activeDocument.findReplace.find(sFindLine))
            {
               UltraEdit.activeDocument.selectLine();
               var sFoundLine = UltraEdit.activeDocument.selection.replace(/[\r\n]+/g,"");

               if (sFoundLine.length == sCurrentLine.length)
               {
                  var nDuplicateLine = UltraEdit.activeDocument.currentLineNum;

                  if (bFirstDuplicate)
                  {
                     UltraEdit.outputWindow.write("\n" + sFullFileName + "(" + nLineNumber + "): " + sCurrentLine);
                     bFirstDuplicate = false;
                  }

                  UltraEdit.outputWindow.write(sFullFileName + "(" + nDuplicateLine + "): " + sFoundLine);

                  for (nIndex = nStartIndex; nIndex < anDuplicateLines.length; nIndex++)
                  {
                     if (anDuplicateLines[nIndex] > nDuplicateLine) break;
                  }

                  anDuplicateLines.splice(nIndex,0,nDuplicateLine);
                  nDuplicates++;
               }
            }
         }
      }

      nLineNumber++;
      UltraEdit.activeDocument.gotoLine(nLineNumber,1);
      if (UltraEdit.activeDocument.currentLineNum != nLineNumber) break;
   }

   UltraEdit.outputWindow.write("\nDuplicate check complete.");
   UltraEdit.outputWindow.showWindow(true);

   if (nDuplicates > 0)
   {
      UltraEdit.outputWindow.write(nDuplicates + " duplicate line" + ((nDuplicates > 1) ? "s" : "") + " found in this file.");
   }
   else
   {
      UltraEdit.outputWindow.write("No duplicate lines found in this file.");
   }
}

Well, for small files it would be definitely better to load entire file into memory into an array of strings and do search for duplicate lines using this array. That would be faster because of no window update during script execution.

Peter · Apr 23, 2018#32018-04-23T07:27+00:00

Great, Mofi - thanks.

Will the old file on the download page be replaced with your new one?

Mofi · Apr 23, 2018#42018-04-23T16:12+00:00

I could improve the code further and comment it and then I could send it to IDM for uploading it on their server. But at the moment I am updating the scripts, macros and wordfiles contributed by me in the past for the downloads pages. I will create a new forum topic if I really come to decision contributing a find duplicates script for UltraEdit user community.

Gabarito · Apr 24, 2018#52018-04-24T01:12+00:00

I'm also interest to see new versions of findDupLines.js.
As I could see, there are a simple and a complex version for that task.
Output log for complex version is far better to analyse the results.
I think that would be a good opportunity to unify those versions.

But it would be very interesting to see also a script to remove duplicate lines that are not successive.
About this, I second the opinion that the script found at Script page has bugs and give me the message "No duplicates found in this file; no lines removed!" even when there are several duplicate lines on the file.

Mofi's suggestion of Perl RegExp is very handy and do the right job successfully.

But how to remove duplicate lines that aren't successive?
I suspect that a couple of changes on findDupLines.js (complex version) is enough to achieve that.

Mofi · Apr 24, 2018#62018-04-24T05:02+00:00

Here is the script modified to remove all duplicates in file with exception of empty lines.

Code: Select all

// This is a simple script which removes duplicate lines.

// THE DUPLICATE CHECK IS CASE SENSITIVE!

// Change the following variable to 0 to disable case sensitivity.

var nCaseSensitive = 1;

if (UltraEdit.document.length > 0)
{
   UltraEdit.insertMode();
   if (typeof(UltraEdit.columnModeOff) == "function") UltraEdit.columnModeOff();
   else if (typeof(UltraEdit.activeDocument.columnModeOff) == "function") UltraEdit.activeDocument.columnModeOff();
   UltraEdit.activeDocument.top();

   UltraEdit.ueReOn();
   UltraEdit.activeDocument.findReplace.mode=0;
   UltraEdit.activeDocument.findReplace.matchCase=false;
   UltraEdit.activeDocument.findReplace.matchWord=false;
   UltraEdit.activeDocument.findReplace.regExp=false;
   UltraEdit.activeDocument.findReplace.searchDown=true;
   if (typeof(UltraEdit.activeDocument.findReplace.searchInColumn) == "boolean")
   {
      UltraEdit.activeDocument.findReplace.searchInColumn=false;
   }

   UltraEdit.outputWindow.write("Beginning duplicate removal...");

   if (nCaseSensitive)
   {
      UltraEdit.activeDocument.findReplace.matchCase = true;
      UltraEdit.outputWindow.write("Case sensitivity enabled");
   }
   else
   {
      UltraEdit.activeDocument.findReplace.matchCase = false;
      UltraEdit.outputWindow.write("Case sensitivity disabled");
   }

   var nDuplicates = 0;
   var nLineNumber = 1;
   var sFullFileName = UltraEdit.activeDocument.path;

   while (!UltraEdit.activeDocument.isEof())
   {
      UltraEdit.activeDocument.selectLine();
      var sCurrentLine = UltraEdit.activeDocument.selection.replace(/[\r\n]+/g,"");

      if (sCurrentLine.length)
      {
         var nLineDuplicates = 0;
         var sFindLine = sCurrentLine.replace(/\^/g,"^^");
         while (UltraEdit.activeDocument.findReplace.find(sFindLine))
         {
            UltraEdit.activeDocument.selectLine();
            var sFoundLine = UltraEdit.activeDocument.selection.replace(/[\r\n]+/g,"");

            if (sFoundLine.length == sCurrentLine.length)
            {
               UltraEdit.activeDocument.deleteText();
               nLineDuplicates++;
               nDuplicates++;
            }
         }
         if (nLineDuplicates)
         {
            UltraEdit.outputWindow.write("\n" + sFullFileName + "(" + nLineNumber + "): " + sCurrentLine);
            UltraEdit.outputWindow.write("Removed " + nLineDuplicates + " duplicate" + ((nLineDuplicates > 1) ? "s" : "") +
                                         " of this line");
         }
      }

      nLineNumber++;
      UltraEdit.activeDocument.gotoLine(nLineNumber,1);
      if (UltraEdit.activeDocument.currentLineNum != nLineNumber) break;
   }

   UltraEdit.outputWindow.write("\nDuplicate removal complete.");
   UltraEdit.outputWindow.showWindow(true);

   if (nDuplicates > 0)
   {
      UltraEdit.outputWindow.write(nDuplicates + " duplicate line" + ((nDuplicates > 1) ? "s" : "") + " removed from this file.");
   }
   else
   {
      UltraEdit.outputWindow.write("No duplicate lines found in this file.");
   }
}

Gabarito · Apr 24, 2018#72018-04-24T09:48+00:00

Thank you, Mofi.
Your script works very well.

Comparing it with findDupLine.js, I could see too many changes and I would not be able to do that alone.