import sys
import os.path
import copy
import codecs

# cleaning GEDCOM files
# by Kaj Holmberg

pri=0

if len(sys.argv)>1:
    infile=sys.argv[1]
else:
    infile="test1.ged"

fname=os.path.basename(sys.argv[1])
print('Cleaning file '+str(fname))
fname=os.path.splitext(fname)[0]
futname=fname+'-c.ged'

text_file=codecs.open(infile,'r',encoding='utf-8',errors='ignore')
lines=text_file.readlines()
text_file.close()
my_count=len(lines)
start=0
print(str(my_count)+' lines read')

print('Cleaning data on file '+str(futname))
#outfile=open(futname,'w')
newlines=[]
keepdel=0
newlines.append('0 HEAD')
for i in range(my_count):
    skip=0
    line1 = lines[i].rstrip()
    line2 = line1.split()
    if len(line2)>1:
        ix=line2[0].strip()
        type1=line2[1]
        if (not ix.isnumeric()) or (len(type1)<2):
            skip=1
        else:
            ix=int(ix)
            type1=line2[1]

            if ix!=1:
                keepdel=0

            # What lines to remove:
            #   keepdel=1 means keep on deleting subsequent rows
            if type1=='_UPD':
                if pri: print('Skip '+line1)
                skip=1
            elif (type1=='CONC' and keepdel==1):
                skip=1
            elif (type1=='CONT' and keepdel==1):
                skip=1
            elif ix>9:
                skip=1
            elif 'Individ tillagd genom' in line1:
                skip=1
            elif 'Fotografi lades till som Smart Match' in line1:
                skip=1
            elif 'Photo was added as Smart Match' in line1:
                skip=1
            elif 'Poster som ej importerats' in line1:
                keepdel=1
                skip=1
            elif 'Poster, som ej importerats' in line1:
                keepdel=1
                skip=1
            elif 'gick inte att förstå' in line1:
                keepdel=1
                skip=1
            elif 'så den ignorerades' in line1:
                keepdel=1
                skip=1
            elif 'Hoppade över underordnad' in line1:
                keepdel=1
                skip=1
            elif 'Tagg upptäckt men stöds ej' in line1:
                keepdel=1
                skip=1
            elif 'men stöds ej' in line1:
                keepdel=1
                skip=1
            elif 'underordnad rad' in line1:
                keepdel=1
                skip=1
            elif 'Tillagd via en' in line1:
                skip=1
            elif 'Tillagd genom ' in line1:
                skip=1
            else:
                if pri: print('Keep line '+line1) 

                # What to replace, for example html-commands
                line1=line1.replace('&amp;lt;br&amp;gt;','. ')
                line1=line1.replace('&amp;lt;p&amp;gt;','. ')
                line1=line1.replace('&amp;lt;p dir="ltr" style="text-align: left;" &amp;gt;','')
                line1=line1.replace('&amp;lt;p style="text-align: left;" dir="ltr" &amp;gt;','')
                line1=line1.replace('&amp;lt;','')
                line1=line1.replace('&amp;gt;','')
                line1=line1.replace('&lt;br&gt;','. ')
                line1=line1.replace('&lt;','')
                line1=line1.replace('&gt;','')
                line1=line1.replace('<table class="recordFieldsTable  ">','')
                line1=line1.replace('</td>','')
                line1=line1.replace('<i>','')
                line1=line1.replace('</i>',' ')
                line1=line1.replace('<b>','')
                line1=line1.replace('</b>',' ')
                line1=line1.replace('<em>','')
                line1=line1.replace('</em>',' ')
                line1=line1.replace('<p>','')
                line1=line1.replace('</p>',' ')
                line1=line1.replace('<span>','')
                line1=line1.replace('</span>',' ')
                line1=line1.replace('<div>',' ')
                line1=line1.replace('</div>',' ')
                line1=line1.replace('<br />',' ')
                line1=line1.replace('<strong>','')
                line1=line1.replace('</strong>','')
                line1=line1.replace('<p style="text-align: left;" dir="ltr">','')
                line1=line1.replace('<p dir="ltr" style="text-align: left;" dir="ltr">','')
                line1=line1.replace('<p dir="ltr" style="text-align: left;">','')
                line1=line1.replace('&quot;','"')
                # Swedish letters and words
                line1=line1.replace('&aring;','å')
                line1=line1.replace('&ouml;','ö')
                line1=line1.replace('&uuml;','u')
                line1=line1.replace('&auml;','ä')
                line1=line1.replace('&oslash;','ö')
                line1=line1.replace('&Aring;','Å')
                line1=line1.replace('&Ouml;','Ö')
                line1=line1.replace('&Oslash;','Ö')
                line1=line1.replace('&Auml;','Ä')
                line1=line1.replace('&AElig;','AE')
                line1=line1.replace('&#039;','e')
                line1=line1.replace('&nbsp;',' ')
                line1=line1.replace('&ndash;','-')
                line1=line1.replace('*','')

                # language
                line1=line1.replace('of Sweden','av Sverige')
                line1=line1.replace('Of Sweden','av Sverige')
                line1=line1.replace('Sweden','Sverige')
                line1=line1.replace('of Denmark','av Danmark')
                line1=line1.replace('Denmark','Danmark')
                line1=line1.replace('of Norway','av Norge')
                line1=line1.replace('Norway','Norge')
                line1=line1.replace('Spouse:','Make/maka:')
                line1=line1.replace('Puoliso:','Make/maka:')

                if 'DATE' in line1:
                    # Gramps doesn't like some date specifications
                    line1=line1.replace('DATE FROM','DATE AFT')
                    line1=line1.replace('DATE Efter','DATE AFT')
                    line1=line1.replace('DATE TO','DATE BEF')
                    line1=line1.replace('DATE Ungefär','DATE ABT')
                    line1=line1.replace('DATE ca','DATE ABT')
                    if ' BC ' in line1:
                        if 'BC' in line2[2]:
                            line1=line2[0]+' '+line2[1]+' '+line2[3]+' B.C.'

    else:
        skip=1

    if line1!='' and skip==0:
        if pri: print('Keep line '+line1) 
        newlines.append(line1)
    else:
        if pri: print('Skip '+line1)

print('Writing cleaned data on file '+str(futname))
outfile=open(futname,'w')
for line in newlines:
    outfile.write(line+'\n')
outfile.close()