import java.io.*;

import java.net.*;

import java.util.*;

import java.util.function.*;

import java.util.logging.Level;

import java.util.logging.Logger;



/*

Changelog 1/8/2017:

- Improved capabilities of program to find obsolete EC entries and replace them with the new EC if available

- If the new EC of an obsolete EC is already in the EC list, KEGG Miner will no longer include both entries preventing duplicate entries

- KEGG Miner no longer includes EC entries representing one organism into the data file

- Added a reporter system to the addition of ECs to the export file for debugging

*/





public class DataMiner {

//Global Instantiated Variables

private final String COMPARE_LIST = "BPI:UDI:SMUP:SUM:SMS:SMG:CHE:BRE:BBU:ZIN:NDL:TPQ:SSDC:HCI:LAS:BHE:BQU:APH:ECH:WPI:WCL:WOL:WRI:OTT:RPQ:RFE:RMC:RAM:FTT:PLR:PLY:CRP:BCI:CEY:CEA:ASY:YPG:MEN:RIP:HDE:BEN:BFL:BCC:BUC:WBR:SGL:PES:SCJ:SLL:SAPI:OHO:FTE:FIN:ZPR:ALM:PKO:FLI:CHU:SLR:SCD:SSM:SBU:AKA:TBD:CVI:NEU:CCS:ELI:PHL:OCO:HEL:HAA:SON:ECO:PFQ:MBJ:ACL:";

private DataMinerInterface DMI;

private boolean includeInList;



private ArrayList<String> VitaminIDs;



public DataMiner(DataMinerInterface DMI){

    this.DMI = DMI;

}



//Advanced Methods

public void mineKegg(File[] files) throws Exception{

    PrintWriter writer2 = null;

    

    for (int i = 0; i < files.length; i++){

        //Instatiates arraylists and wipes used arraylists

        ArrayList<String> setAbbrv = new ArrayList();

        ArrayList<String> excel = new ArrayList();

        ArrayList<String> onesandzeros = new ArrayList();

        VitaminIDs = new ArrayList();

        ArrayList<String> ECs = new ArrayList();

                

        //<editor-fold defaultstate="collapsed" desc="adds compareList into setAbbrv and excel arrayLists">

        String processedString = "";



        for (int k = 0; k<COMPARE_LIST.length(); k++){

            if( COMPARE_LIST.charAt(k) == ':'){

                setAbbrv.add(processedString);

                excel.add(processedString);

                processedString = "";

                //System.out.println(processedString);

            }

            else

            {

                processedString = processedString + COMPARE_LIST.charAt(k);

                //System.out.println(processedString);

            }

        }

        //</editor-fold>

        

        //Generates file reader

        FileReader file = new FileReader(files[i]);

        BufferedReader in = new BufferedReader(file);

        String inputLine;

        

        //Adds all ECs to be processed from text file

        String filename = files[i].toString();

        //System.out.println(filename + "\n===============================");

        DMI.addText("\n" + filename + "\n=======================================\n");

        while ((inputLine = in.readLine()) != null){

            

            inputLine = inputLine.replace("�", ""); //Removes anamolous characters from input text files

            inputLine = inputLine.replace(" ", "");

            //System.out.println(inputLine);        //Debugging EC printer

            DMI.addText(inputLine + "\n");

            VitaminIDs.add(inputLine);

        }

        //System.out.println(VitaminIDs);       //Debugging VitaminID arraylist additions

        in.close();

        

        //Actual data mining process

        for(int j = 0; j<VitaminIDs.size(); j++){

        //for( int j = 0; j<VitaminIDs.size(); j++){

            //String organismList = getFromWeb(new URL("https://urldefense.proofpoint.com/v2/url?u=http-3A__www.genome.jp_dbget-2Dbin_www-5Fbget-3Fec-3A&d=AwIGaQ&c=1QsCMERiq7JOmEnKpsSyjg&r=j9BjNmxXIwCVhVOItuh4qA&m=DKh9xIxhDhnzC8qOe7BJ_a7V6hroczkaCz_-G2-e5e0&s=EaNzOsjtadtEvrc-I5OncGSn6dQ4PHASzLL15J2dQRI&e= "+VitaminIDs.get(j)));

            String organismList = getFromWeb(new URL("https://urldefense.proofpoint.com/v2/url?u=http-3A__www.genome.jp_dbget-2Dbin_www-5Fbget-3Fec-3A&d=AwIGaQ&c=1QsCMERiq7JOmEnKpsSyjg&r=j9BjNmxXIwCVhVOItuh4qA&m=DKh9xIxhDhnzC8qOe7BJ_a7V6hroczkaCz_-G2-e5e0&s=EaNzOsjtadtEvrc-I5OncGSn6dQ4PHASzLL15J2dQRI&e= "+VitaminIDs.get(j)));

            if (includeInList){

                ECs.add(VitaminIDs.get(j)); //Adds found EC to list of ECs

                processData(organismList, setAbbrv, onesandzeros);

                addOutcomes(onesandzeros, excel);

                int sum = 0;

                for (int z = 0; z < onesandzeros.size(); z++){

                    sum+=Integer.parseInt(onesandzeros.get(z));

                }

                DMI.addText(sum + " match(es) found in " + VitaminIDs.get(j) + "\n");

                onesandzeros = new ArrayList();

                double progress = ((double)(j+1)/(double)VitaminIDs.size())*100;

                DMI.getProgressBar().setValue((int)progress);

                //System.out.println(progress);

                export(filename.substring(0,filename.indexOf(".txt")) + " output.txt", excel, ECs);

            }

            else{

                double progress = ((double)(j+1)/(double)VitaminIDs.size())*100;

                DMI.addText(VitaminIDs.get(j) + " was not included\n");

                DMI.getProgressBar().setValue((int)progress);

                

                try 

                {

                    if(writer2==null){

                        writer2 = new PrintWriter(new File("ExcludedECs.txt"));

                    }

                    writer2.println(VitaminIDs.get(j));

                }

                catch(Exception e)

                {}

            }

        }

    

    }

    writer2.close();

    DMI.getStartButton().setEnabled(true);

}



public void minePDF (File[] files, boolean isNAD) throws Exception{

    for (int i = 0; i < files.length; i++){

        FileReader file = new FileReader(files[i]);

        if(isNAD)

            export(files[i].toString().substring(0, files[i].toString().indexOf(".pdf"))+" NAD EC scan output.txt", grabECsFromPDF(file, isNAD));

        else

            export(files[i].toString().substring(0, files[i].toString().indexOf(".pdf"))+" NADP EC scan output.txt", grabECsFromPDF(file, isNAD));

    }

    DMI.getStartButton().setEnabled(true);

}

    

//Functional Components

public String getFromWeb(URL url) throws Exception{

        String output = "";

        includeInList = true;

        url = checkForObsoleteEnzyme(url);  //repairs obsolescence

        

        if(includeInList){

        //Generates web reader

        BufferedReader in = new BufferedReader(

        new InputStreamReader(url.openStream()));

        String inputLine;

        

        //Parses through web page and acquires data

        includeInList = false;

        while ((inputLine = in.readLine()) != null)

            if((inputLine.contains("<input type=\"hidden\" name=\"query\" value="))){

                output = inputLine.substring(41);

                output = output.substring(0,output.indexOf("\""));

                includeInList = true;

            }

        in.close();

        }

        return output;

    }

    

public void processData(String input, ArrayList<String> setAbbrv, ArrayList<String> onesandzeros){

        //Generates temporary array list of collected data

        ArrayList<String> collAbbrv = new ArrayList();

        //Adds output to ArrayList

        String processedString = "";

        for (int i = 0; i<input.length(); i++){

            if(input.charAt(i) == ' '){

                collAbbrv.add(processedString);

                processedString = "";

            }

            else

            {

                processedString = processedString + input.charAt(i);

            }

        }



        //Performs cross-analysis between SetAbbrv and collAbbrv

        String currentAbbrv = "";

        boolean hit;

        for (int i = 0; i<setAbbrv.size(); i++){



            hit = false;

            currentAbbrv = setAbbrv.get(i);

            for (int k = 0; k<collAbbrv.size(); k++){

                if(currentAbbrv.equals(collAbbrv.get(k))){

                    onesandzeros.add("1");

                    hit = true;

                    break;

                }

            }

            if (!hit){

                onesandzeros.add("0");

            }

        }

    }



public void addOutcomes(ArrayList onesandzeros, ArrayList excel){

        for (int i = 0; i<onesandzeros.size(); i++){

            excel.set(i, excel.get(i) + " " + onesandzeros.get(i));

        }

    }



public ArrayList<String> grabECsFromPDF(FileReader file, boolean isNAD) throws Exception{

        ArrayList<String> legitECs = new ArrayList();

        BufferedReader in = new BufferedReader(file);

        String docInputLine;

        URL url;



        while ((docInputLine = in.readLine()) != null){

            if (docInputLine.contains("(https://urldefense.proofpoint.com/v2/url?u=http-3A__www.enzyme-2Ddatabase.org_query.php-3Fec-3D&d=AwIGaQ&c=1QsCMERiq7JOmEnKpsSyjg&r=j9BjNmxXIwCVhVOItuh4qA&m=DKh9xIxhDhnzC8qOe7BJ_a7V6hroczkaCz_-G2-e5e0&s=L4qsGW7chShEN_HI9asJIC2Bq-IVA_9gTdnorch6JR4&e= ")){

                url = new URL(docInputLine.substring(docInputLine.indexOf("http"),docInputLine.indexOf(")>>")));

                if(webParser(url,isNAD)){

                    DMI.addText(docInputLine.substring(docInputLine.indexOf("ec=")+3,docInputLine.indexOf(")>>")));

                    DMI.addText("\n\n");

                    legitECs.add(docInputLine.substring(docInputLine.indexOf("ec=")+3,docInputLine.indexOf(")>>")));

                }

            }

        }

        in.close();

        return legitECs;      

}



public ArrayList<String> grabECsFromList() throws Exception{

        ArrayList<String> legitECs = new ArrayList();

        FileReader file = new FileReader("C:\\Users\\Brian\\Documents\\NetBeansProjects\\Programming\\input.txt");

        BufferedReader in = new BufferedReader(file);

        String docInputLine;

        URL url;

        

        //Parses through local document

        while ((docInputLine = in.readLine()) != null){

            url = new URL("https://urldefense.proofpoint.com/v2/url?u=http-3A__www.enzyme-2Ddatabase.org_query.php-3Fec-3D&d=AwIGaQ&c=1QsCMERiq7JOmEnKpsSyjg&r=j9BjNmxXIwCVhVOItuh4qA&m=DKh9xIxhDhnzC8qOe7BJ_a7V6hroczkaCz_-G2-e5e0&s=L4qsGW7chShEN_HI9asJIC2Bq-IVA_9gTdnorch6JR4&e= "+docInputLine);

                

            if(webParser(url,true)){

                DMI.addText(docInputLine + "\n");

                legitECs.add(docInputLine);



            }

                



            }

        in.close();

        return legitECs;      

    }



public ArrayList<String> scanSpecialCases(File[] files, boolean isNAD) throws Exception{

        ArrayList<String> legitECs = new ArrayList();

        

        

        for(File file : files){

            FileReader f = new FileReader(file);

            BufferedReader in = new BufferedReader(f);

            String docInputLine;

            URL url;

            while ((docInputLine = in.readLine()) != null){

                if (docInputLine.contains("(https://urldefense.proofpoint.com/v2/url?u=http-3A__www.enzyme-2Ddatabase.org_query.php-3Fec-3D&d=AwIGaQ&c=1QsCMERiq7JOmEnKpsSyjg&r=j9BjNmxXIwCVhVOItuh4qA&m=DKh9xIxhDhnzC8qOe7BJ_a7V6hroczkaCz_-G2-e5e0&s=L4qsGW7chShEN_HI9asJIC2Bq-IVA_9gTdnorch6JR4&e= ")){

                    url = new URL(docInputLine.substring(docInputLine.indexOf("http"),docInputLine.indexOf(")>>")));

                    if(webParserSpecialCases(url,isNAD)){

                        DMI.addText(docInputLine.substring(docInputLine.indexOf("ec=")+3,docInputLine.indexOf(")>>")));

                        DMI.addText("\n\n");

                        legitECs.add(docInputLine.substring(docInputLine.indexOf("ec=")+3,docInputLine.indexOf(")>>")));

                    }

                }

            }

            export(file.toString().substring(0,file.toString().indexOf(".pdf")) + " special cases output.txt",legitECs);

            in.close();

        }

        DMI.getStartButton().setEnabled(true);

        

        return legitECs;      

}



public void export(String filename, ArrayList excel){

        try{

            PrintWriter writer = new PrintWriter(filename, "UTF-8");

            for (int i = 0; i<excel.size(); i++){

                writer.println(excel.get(i));

            }

            writer.close();

        } catch (IOException e) {

        }   

    }



//Method overload allowing for the addition of only found ECS in the MineKegg function

public void export(String filename, ArrayList excel, ArrayList ECs){

        try{

            PrintWriter writer = new PrintWriter(filename, "UTF-8");

            

            for (int j = 0; j<ECs.size(); j++){

                writer.print(" "+ECs.get(j));

            }

            writer.println();

            for (int i = 0; i<excel.size(); i++){

                writer.println(excel.get(i));

            }

            writer.close();

        } catch (IOException e) {

        }   

    }



//Utility

//Error Checking Component

private URL checkForObsoleteEnzyme(URL url) throws Exception {

        //Instantiates relevant variables

        String newNumber = "";

        boolean obsolete = false;

        URL newURL = url;

        

        //Generates web reader with predetermined URL

        BufferedReader in = new BufferedReader(

        new InputStreamReader(url.openStream()));

        String inputLine;

        //Parses through web

        while ((inputLine = in.readLine()) != null)

        {

            //Checks for the word Obsolete indicative of a webpage that has been moved

            if((inputLine.contains("Obsolete"))){

                obsolete = true;

            }

            if(obsolete&&inputLine.contains("ec:")){

                newNumber = inputLine;

                newNumber = newNumber.substring(newNumber.indexOf("ec:")+3);

                newNumber = newNumber.substring(0,newNumber.indexOf("\""));

                

                //Renames EC to new number if that number does not already exist

                int index = VitaminIDs.indexOf(VitaminIDs.indexOf(url.toString().substring(url.toString().indexOf("ec:") + 3)));

                

                if(!VitaminIDs.contains(newNumber)){

                includeInList = true;

                VitaminIDs.set(VitaminIDs.indexOf(url.toString().substring(url.toString().indexOf("ec:") + 3)), newNumber);

                DMI.addText(url.toString().substring(url.toString().indexOf("ec:") + 3) + " is obsolete\n");

                DMI.addText("New EC is " + newNumber + "\n");

                DMI.addText("New EC is not already present in the list and has been added.\n");

                }

                else{

                includeInList = false;

                DMI.addText(url.toString().substring(url.toString().indexOf("ec:") + 3) + " is obsolete\n");

                DMI.addText("New EC is " + newNumber + "\n");

                DMI.addText(VitaminIDs.get(VitaminIDs.indexOf(newNumber))+ " already exists in EC List, obsolete EC will be ignored.\n");

                }

                

                newURL = new URL("https://urldefense.proofpoint.com/v2/url?u=http-3A__www.genome.jp_dbget-2Dbin_www-5Fbget-3Fec-3A&d=AwIGaQ&c=1QsCMERiq7JOmEnKpsSyjg&r=j9BjNmxXIwCVhVOItuh4qA&m=DKh9xIxhDhnzC8qOe7BJ_a7V6hroczkaCz_-G2-e5e0&s=EaNzOsjtadtEvrc-I5OncGSn6dQ4PHASzLL15J2dQRI&e= " + newNumber);

                break;

                //overly strict scanner

                /*if(inputLine.contains("Now covered by EC") || inputLine.contains("Now classified as EC") || inputLine.contains("Transferred to")){

                        newNumber = inputLine;

                        newNumber = newNumber.substring(newNumber.indexOf("ec:")+3);

                        newNumber = newNumber.substring(0,newNumber.indexOf("\""));

                        url = new URL("https://urldefense.proofpoint.com/v2/url?u=http-3A__www.genome.jp_dbget-2Dbin_www-5Fbget-3Fec-3A&d=AwIGaQ&c=1QsCMERiq7JOmEnKpsSyjg&r=j9BjNmxXIwCVhVOItuh4qA&m=DKh9xIxhDhnzC8qOe7BJ_a7V6hroczkaCz_-G2-e5e0&s=EaNzOsjtadtEvrc-I5OncGSn6dQ4PHASzLL15J2dQRI&e= " + newNumber);

                }*/

            }

        }

        in.close();

        

        //Ensures that a new URL has been acquired

        if(!newURL.equals(url)&&obsolete){

            return newURL;

        }

        else if(!obsolete){

            return url;

        }

        else{

            DMI.addText("!!! Failed to acquire new URL despite obsolescence !!!\n");

            includeInList = false;

            return url;

        }

    }



//Data moving components for ECGrabbers

private boolean webParser(URL url, boolean isNAD) throws Exception{

    boolean isParser = true;

    

    //Generates web reader

    BufferedReader wr = new BufferedReader(

    new InputStreamReader(url.openStream()));

    String webInputLine;



    

    //Parses through web page and acquires data

    while ((webInputLine = wr.readLine()) != null){

        if(isNAD){

                if(NAD_Criterion(webInputLine,isParser)){

                    DMI.addText("Line of Evidence:\n");

                    DMI.addText(webInputLine+"\n");

                    return true;

                }

        }

        else{

                if(NADP_Criterion(webInputLine,isParser)){

                    DMI.addText("Line of Evidence:\n");

                    DMI.addText(webInputLine+"\n");

                    return true;

                }

        }

    }

//</editor-fold>

        wr.close();

        return false; 

    }   //Reads through https://urldefense.proofpoint.com/v2/url?u=http-3A__www.enzyme-2Ddatabase.org_query.php-3Fec-3Dx.x.x.x&d=AwIGaQ&c=1QsCMERiq7JOmEnKpsSyjg&r=j9BjNmxXIwCVhVOItuh4qA&m=DKh9xIxhDhnzC8qOe7BJ_a7V6hroczkaCz_-G2-e5e0&s=V6fwekzSSYEc2Uvs3LN7egQmvlaJ42YNjmxpjvk9ClM&e=  for hits of scripted target !!!Must be rewritten!!! 



private boolean webParserSpecialCases(URL url,boolean isNAD) throws Exception{

    //Generates web reader

    boolean isParser = false;

    BufferedReader wr = new BufferedReader(

    new InputStreamReader(url.openStream()));

    String webInputLine;





    //Parses through web page and acquires data

    while ((webInputLine = wr.readLine()) != null){

        if(isNAD){

                if(NAD_Criterion(webInputLine,isParser)){

                    DMI.addText("Special Case:\n");      //Debug code for checking if claim to NAD dependence is legitimized by the presence of NADP as a reactent

                    DMI.addText(webInputLine + "\n");

                    return true;

                }

        }

        else{

                if(NADP_Criterion(webInputLine,isParser)){

                    DMI.addText("Special Case:\n");      //Debug code for checking if claim to NADP dependence is legitimized by the presence of NADP as a reactent

                    DMI.addText(webInputLine + "\n");

                    return true;

                }

        }

    }

//</editor-fold>

        wr.close();

        return false; 

    }  //Scans for special cases in ones ECs that would fail the normal criterions



//Advanced Boolean Handlers

private boolean NADP_Criterion(String webInputLine, boolean isParser){

    

    if(isParser){

        if(webInputLine.contains("<b>Reaction:</b>")&&((webInputLine.contains("NADP"))||(webInputLine.contains("NAD(P)")))){

            return true;

        }

        if((webInputLine.contains("<b>Comments:</b>"))&&(webInputLine.contains("Requires NADP")||webInputLine.contains("Requires NAD(P)"))){

            return true;

        }

    }

    

    else if (!isParser){

        if(webInputLine.contains("<b>Reaction:</b>")&&((webInputLine.contains("NADP")||webInputLine.contains("NAD(P)")))){

            return false;

        }

        if((webInputLine.contains("<b>Comments:</b>"))&&(webInputLine.contains("NADP")||webInputLine.contains("NAD(P)"))&&!(webInputLine.contains("Requires NADP")||webInputLine.contains("Requires NAD(P)")))

        {

            return true;

        }

    }

    return false;

}



private boolean NAD_Criterion(String webInputLine, boolean isParser){

    

    if(isParser){

        if(webInputLine.contains("<b>Reaction:</b>")&&(webInputLine.contains("NAD"))&&(!webInputLine.contains("NADP"))){

            return true;

        }

        if((webInputLine.contains("<b>Comments:</b>"))&&(webInputLine.contains("Requires NAD"))  

        &&!(webInputLine.contains("Requires NADP"))){

            return true;

        }

    }

    

    else if (!isParser) {

        if(webInputLine.contains("<b>Reaction:</b>")&&(webInputLine.contains("NAD"))&&(!webInputLine.contains("NADP"))){

            return false;

        }

        if((webInputLine.contains("<b>Comments:</b>"))&&(webInputLine.contains("NAD"))&&!(webInputLine.contains("Requires NAD")))

        {

            return true;

        }

    }

    return false;

}



}