Hi,
Need your help!!!
I have a text file.I need to check whether there are duplicate records in this file for the field'MVTJOU_NEW value".If there are the duplicate records in the file ,then that row should be deleted and a new output file should be created without duplicates.Each row starts with a reference number which is unique i.e B4A31FCHIT100000,B4A31FCHIT100001 etc..
0;B4A31FCHIT100000;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100000;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="12182" /><DPTCOD value="013945" /><PDTCOD value="000401" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="3187518.07" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100001;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100001;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="12184" /><DPTCOD value="013945" /><PDTCOD value="000401" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100002;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100002;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="22005" /><DPTCOD value="013267" /><PDTCOD value="000318" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="-219.09" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100003;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100003;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="23061" /><DPTCOD value="013267" /><PDTCOD value="000118" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100004;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100004;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="23061" /><DPTCOD value="013945" /><PDTCOD value="000401" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100005;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100005;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="28413" /><DPTCOD value="013945" /><PDTCOD value="000166" /><AFFCOD value="GM141" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100006;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100006;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="28416" /><DPTCOD value="013945" /><PDTCOD value="000166" /><AFFCOD value="GM141" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="-3190817.96" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100007;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100007;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="28475" /><DPTCOD value="013945" /><PDTCOD value="000166" /><AFFCOD value="GM141" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="1656.65" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100008;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100008;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="31054" /><DPTCOD value="013267" /><PDTCOD value="000411" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100009;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100009;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="41310" /><DPTCOD value="013945" /><PDTCOD value="000166" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="-0.02" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100010;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100010;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="41312" /><DPTCOD value="013267" /><PDTCOD value="000118" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100011;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100011;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="41312" /><DPTCOD value="013945" /><PDTCOD value="000401" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100012;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100012;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="41312" /><DPTCOD value="013267" /><PDTCOD value="000411" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100013;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100013;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="41316" /><DPTCOD value="013945" /><PDTCOD value="000401" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="-3187518.07" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100014;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100014;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="41318" /><DPTCOD value="013945" /><PDTCOD value="000401" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100015;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100015;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="41326" /><DPTCOD value="013267" /><PDTCOD value="000411" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100016;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100016;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="41332" /><DPTCOD value="013945" /><PDTCOD value="000166" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="3189161.31" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100017;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100017;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="50461" /><DPTCOD value="013267" /><PDTCOD value="000318" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="219.09" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100018;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100018;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="12182" /><DPTCOD value="013945" /><PDTCOD value="000401" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="3187518.07" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100019;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100019;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="12184" /><DPTCOD value="013945" /><PDTCOD value="000401" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100020;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100020;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="22005" /><DPTCOD value="013267" /><PDTCOD value="000318" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="-219.09" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100021;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100021;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="23061" /><DPTCOD value="013267" /><PDTCOD value="000118" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100022;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100022;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="23061" /><DPTCOD value="013945" /><PDTCOD value="000401" /><AFFCOD value="" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>
0;B4A31FCHIT100023;20140131;20140131;12000000;HIT;HITINT;B4A31FCHIT100023;2000;ZAA2; ; ; ; ;<?xml version="1.0" encoding="ISO-8859-1" ?><XITP><SET number="1"><BUSUNI value="CB464" /><ACCNBR
value="28413" /><DPTCOD value="013945" /><PDTCOD value="000166" /><AFFCOD value="GM141" /><DEVCPT value="AUD" /><NUMCLI value="" /><NUMCPT value="" /></SET><SET
number="2"><DATCRA_NEW value="30-01-2014" /><MVTJOU_NEW value="0.00" /><SWIENV_NEW value="N" /></SET></XITP></rec>