Remove occurrences of duplicate words in a string

Question

Take the following string as an example:

var string = "spanner, span, spaniel, span";

From this string I would like to find the duplicate words, remove all the duplicates keeping one occurrence of the word in place and then output the revised string.

Which in this example would be:

var string = "spanner, span, spaniel";

I've setup a jsFiddle for testing: http://jsfiddle.net/p2Gqc/

Note that the order of the words in the string is not consistent, neither is the length of each string so a regex isn't going to do the job here I don't think. I'm thinking something along the lines of splitting the string into an array? But I'd like it to be as light on the client as possible and super speedy...

Answer 1

How about something like this?

split the string, get the array, filter it to remove duplicate items, join them back.

var uniqueList=string.split(',').filter(function(item,i,allItems){
    return i==allItems.indexOf(item);
}).join(',');

$('#output').append(uniqueList);

Fiddle

For non supporting browsers you can tackle it by adding this in your js.

See Filter

if (!Array.prototype.filter)
{
  Array.prototype.filter = function(fun /*, thisp*/)
  {
    "use strict";

    if (this == null)
      throw new TypeError();

    var t = Object(this);
    var len = t.length >>> 0;
    if (typeof fun != "function")
      throw new TypeError();

    var res = [];
    var thisp = arguments[1];
    for (var i = 0; i < len; i++)
    {
      if (i in t)
      {
        var val = t[i]; // in case fun mutates this
        if (fun.call(thisp, val, i, t))
          res.push(val);
      }
    }

    return res;
  };
}

Answer 2

If non of the above works for you here is another way:

var str = "spanner, span, spaniel, span";
str = str.replace(/[ ]/g,"").split(",");
var result = [];
for(var i =0; i < str.length ; i++){
    if(result.indexOf(str[i]) == -1) result.push(str[i]);
}
result=result.join(", ");

Or if you want it to be in a better shape try this:

Array.prototype.removeDuplicate = function(){
   var result = [];
   for(var i =0; i < this.length ; i++){
       if(result.indexOf(this[i]) == -1) result.push(this[i]);
   }
   return result;
}
var str = "spanner, span, spaniel, span";
str = str.replace(/[ ]/g,"").split(",").removeDuplicate().join(", ");

Answer 3

Alternate Solution using Regular Expression

By making use of positive lookahead, you can strip off all the duplicate words.

Regex /(\b\S+\b)(?=.*\1)/ig, where

• \b - matches word boundary
• \S - matches character that is not white space(tabs, line breaks,etc)
• ?= - used for positive lookahead
• ig - flags for in-casesensitive,global search respectively
• +,* - quantifiers. + -> 1 or more, * -> 0 or more
• () - define a group
• \1 - back-reference to the results of the previous group

var string1 = 'spanner, span, spaniel, span';
var string2 = 'spanner, span, spaniel, span, span';
var string3 = 'What, the, the, heck';
// modified regex to remove preceding ',' and ' ' as per your scenario 
var result1 = string1.replace(/(\b, \w+\b)(?=.*\1)/ig, '');
var result2 = string2.replace(/(\b, \w+\b)(?=.*\1)/ig, '');
var result3 = string3.replace(/(\b, \w+\b)(?=.*\1)/ig, '');
console.log(string1 + ' => ' + result1);
console.log(string2 + ' => ' + result2);
console.log(string3 + ' => ' + result3);

The only caveat is that this regex keeps only the last instance of a found duplicate word and strips off all the rest. To those who care only about duplicates and not about the order of the words, this should work!

Answer 4

In getUniqueWordString function, we are filtering redundant words and then joining back with delimiter. Added one case also if in Input string words exist in Upper and lower case both.

function getUniqueWordString(str, delimiter) {
    return str.toLowerCase().split(delimiter).filter(function(e, i, arr) {
        return arr.indexOf(e, i+1) === -1;
    }).join(delimiter);
}

let str = "spanner, span, spaniel, span, SPAN, SpaNiel";
console.log(getUniqueWordString(str, ", "))

Answer 5

modern approach using Set

let string = "spanner, span, spaniel, span";

let unique = [...new Set(string.split(", ")];

console.log(unique);

Answer 6

// Take the following string
var string = "spanner, span, spaniel, span";
var arr = string.split(", ");
var unique = [];
$.each(arr, function (index,word) {
    if ($.inArray(word, unique) === -1) 
        unique.push(word);

});

alert(unique);

Live DEMO

Answer 7

Both the other answers would work fine, although the filter array method used by PSL was added in ECMAScript 5 and won't be available in old browsers.

If you are handling long strings then using $.inArray/Array.indexOf isn't the most efficient way of checking if you've seen an item before (it would involve scanning the whole array each time). Instead you could store each word as a key in an object and take advantage of hash-based look-ups which will be much faster than reading through a large array.

var tmp={};
var arrOut=[];
$.each(string.split(', '), function(_,word){
    if (!(word in tmp)){
        tmp[word]=1;
        arrOut.push(word);
    }
});
arrOut.join(', ');

Answer 8

<script type="text/javascript">
str=prompt("Enter String::","");
arr=new Array();
arr=str.split(",");
unique=new Array();
for(i=0;i<arr.length;i++)
{
    if((i==arr.indexOf(arr[i]))||(arr.indexOf(arr[i])==arr.lastIndexOf(arr[i])))
        unique.push(arr[i]);   
}
unique.join(",");
alert(unique);
</script>

this code block will remove duplicate words from a sentence.

the first condition of if statement i.e (i==arr.indexOf(arr[i])) will include the first occurence of a repeating word to the result(variale unique in this code).

the second condition (arr.indexOf(arr[i])==arr.lastIndexOf(arr[i])) will include all non repeating words.

Answer 9

below is an easy to understand and quick code to remove duplicate words in a string:

var string = "spanner, span, spaniel, span";


var uniqueListIndex=string.split(',').filter(function(currentItem,i,allItems){
    return (i == allItems.indexOf(currentItem));
});

var uniqueList=uniqueListIndex.join(',');

alert(uniqueList);//Result:spanner, span, spaniel

As simple as this can solve your problem. Hope this helps. Cheers :)

Answer 10

To delete all duplicate words, I use this code:

<script>
function deleteDuplicate(a){a=a.toString().replace(/ /g,",");a=a.replace(/[ ]/g,"").split(",");for(var b=[],c=0;c<a.length;c++)-1==b.indexOf(a[c])&&b.push(a[c]);b=b.join(", ");return b=b.replace(/,/g," ")};
document.write(deleteDuplicate("g g g g"));
</script>

Answer 11

var string = "spanner, span, spaniel, span";

var strArray= string.split(",");

var unique = [];
 for(var i =0; i< strArray.length; i++)
 {
   eval(unique[strArray] = new Object()); 
 }

//You can easily traverse the unique through foreach.

I like this for three reason. First, it works with IE8 or any other browser.

Second. it is more optimized and guaranteed to have unique result.

Last, It works for Other String array which has White space in their inputs like

var string[] = {"New York", "New Jersey", "South Hampsire","New York"};

for the above case there will be only three elements in the string[] which would be uniquely stored.