'''
Created on ١١‏/٠٥‏/٢٠١٠

@Created by: Muhammad Altabba
'''
from Models.Lexicon.RootsAndPatternsRepository import *;
from Models.Lexicon.SpecialWords.StandAloneParticle import *;
from Models.Lexicon.SpecialWords.ProperNoun import *;

from Controllers.TextEntities.TextEncapsulator import *;
from Controllers.TextEntities.Word import *;
from Controllers.Tokenization.Tokenizer import *;
from Controllers.Normalization.Normalizer import *;
from Controllers.Morphology.AffixParser import *;
from Controllers.Morphology.MorphologicalAnalyzer import *;

import codecs;
import io;
import os;
from os.path import join, getsize;



compoundNounsXmlFile = '../../Data/MorphologyTransducers/Proclitics.xml';
procliticsXmlFile = '../../Data/MorphologyTransducers/Proclitics.xml';
encliticsXmlFile = '../../Data/MorphologyTransducers/Enclitics.xml';
prematureTaggingRulesXmlFile = '../../Data/TaggingRepository/PrematureTaggingRules.xml';
overdueTaggingRulesXmlFile = '../../Data/TaggingRepository/OverdueTaggingRules.xml';
baseDirectoryOfAlKhalil = 'D:/temp/AlKhalil_1/db/'
rootsFolder = 'roots2'


text = TextEncapsulator();
text.LoadFromFiles(baseDirectoryOfAlKhalil, rootsFolder, \
                   procliticsXmlFile, encliticsXmlFile,\
                   prematureTaggingRulesXmlFile, \
                   overdueTaggingRulesXmlFile);

base = 'D:/temp/Latifa2/'




for root, dirs, files in os.walk(base):
    for dir in dirs:
        print('Start parsing directory: ['+dir+']');
        for subroot, subdirs, subfiles in os.walk(root+dir):
            for file in subfiles:
                if file.endswith('.txt') and file.find('-') == -1 :  
                    if(file.find('Edu') == -1 ):
                        continue;
                    print('\tStart parsing file: ['+file+']');
                    
                    f = codecs.open('/'.join([subroot, file]), 'r', 'utf-8');
                    string = f.read();
                    f.close();
                    
                    text.String = string;
                    text.Tokenize();
                    text.Normalize(2);
                    
                    text.ParseClitics();
                    
                    print('\tProcessing...');
                    
                    text.PatternMatchingSimpleStem();
                    
                    print('\tWriting...');
                    
                    xmlStreamWriter = io.StringIO();
                    text.RenderTextSimpleStem(xmlStreamWriter);
                    writer = codecs.open('/'.join([subroot, file.replace('.txt','-Qutuf.txt')]), 'w', 'utf-8');
                    writer.write(xmlStreamWriter.getvalue());
                    xmlStreamWriter.close();
                    writer.close();
                    
                    print ('\tEnd parsing file: ['+file+']');
                    print('------------------------------------------------------');