/*
 *    KEAPhraseFilter.java
 *    Copyright (C) 2001 Eibe Frank
 *
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

import java.util.*;

import weka.core.*;
import weka.filters.*;

/**
 * This filter splits the text in selected string
 * attributes into phrases. The resulting
 * string attributes contain these phrases
 * separated by '\n' characters.
 *
 * Phrases are identified according to the
 * following definitions:
 * 
 * A phrase is a sequence of words interrupted
 * only by sequences of whitespace characters,
 * where each sequence of whitespace characters
 * contains at most one '\n'.
 *
 * A word is a sequence of letters or digits
 * that contains at least one letter, with
 * the following exceptions:
 *
 * a) '.', '@', '_', '&', '/', '-' are allowed
 * if surrounded by letters or digits,
 *
 * b) '\'' is allowed if preceeded by a letter
 * or digit,
 * 
 * c) '-', '/' are also allowed if succeeded by
 * whitespace characters followed by another
 * word. In that case the whitespace characters
 * will be deleted.
 *
 * @author Eibe Frank (eibe@cs.waikato.ac.nz)
 * @version 1.0
 */
public class KEAPhraseFilter extends Filter implements OptionHandler {

  /** Stores which columns to select as a funky range */
  protected Range m_SelectCols = new Range();

  /** Determines whether internal periods are allowed */
  protected boolean m_DisallowInternalPeriods = false;

  /**
   * Returns a string describing this filter
   *
   * @return a description of the filter suitable for
   * displaying in the explorer/experimenter gui
   */
  public String globalInfo() {
    return "This filter splits the text contained " +
      "by the selected string attributes into phrases.";
  }

  /**
   * Returns an enumeration describing the available options
   *
   * @return an enumeration of all the available options
   */
  public Enumeration listOptions() {

    Vector newVector = new Vector(3);

    newVector.addElement(new Option(
              "\tSpecify list of attributes to process. First and last are valid\n"
	      +"\tindexes. (default none)",
              "R", 1, "-R <index1,index2-index4,...>"));
    newVector.addElement(new Option(
	      "\tInvert matching sense",
              "V", 0, "-V"));
    newVector.addElement(new Option(
	      "\tDisallow internal periods",
              "P", 0, "-P"));

    return newVector.elements();
  }

  /**
   * Parses a given list of options controlling the behaviour of this object.
   * Valid options are:<p>
   *
   * -R index1,index2-index4,...<br>
   * Specify list of attributes to process. First and last are valid indexes.
   * (default none)<p>
   *
   * -V<br>
   * Invert matching sense <p>
   *
   * -P<br>
   * Disallow internal periods <p>
   *
   * @param options the list of options as an array of strings
   * @exception Exception if an option is not supported
   */
  public void setOptions(String[] options) throws Exception {

    String list = Utils.getOption('R', options);
    if (list.length() != 0) {
      setAttributeIndices(list);
    }
    setInvertSelection(Utils.getFlag('V', options));

    setDisallowInternalPeriods(Utils.getFlag('P', options));
    
    if (getInputFormat() != null) {
      setInputFormat(getInputFormat());
    }
  }

  /**
   * Gets the current settings of the filter.
   *
   * @return an array of strings suitable for passing to setOptions
   */
  public String [] getOptions() {

    String [] options = new String [4];
    int current = 0;

    if (getInvertSelection()) {
      options[current++] = "-V";
    }
    if (getDisallowInternalPeriods()) {
      options[current++] = "-P";
    }
    if (!getAttributeIndices().equals("")) {
      options[current++] = "-R"; options[current++] = getAttributeIndices();
    }

    while (current < options.length) {
      options[current++] = "";
    }
    return options;
  }

  /**
   * Sets the format of the input instances.
   *
   * @param instanceInfo an Instances object containing the input
   * instance structure (any instances contained in the object are
   * ignored - only the structure is required).
   * @return true if the outputFormat may be collected immediately 
   */
  public boolean setInputFormat(Instances instanceInfo) throws Exception {

    super.setInputFormat(instanceInfo);
    setOutputFormat(instanceInfo);
    m_SelectCols.setUpper(instanceInfo.numAttributes() - 1);

    return true;
  }

  /**
   * Input an instance for filtering. Ordinarily the instance is processed
   * and made available for output immediately. Some filters require all
   * instances be read before producing output.
   *
   * @param instance the input instance
   * @return true if the filtered instance may now be
   * collected with output().
   * @exception Exception if the input instance was not of the correct 
   * format or if there was a problem with the filtering.
   */
  public boolean input(Instance instance) throws Exception {

    if (getInputFormat() == null) {
      throw new Exception("No input instance format defined");
    }
    if (m_NewBatch) {
      resetQueue();
      m_NewBatch = false;
    }
    convertInstance(instance);
    return true;
  }

  /**
   * Signify that this batch of input to the filter is finished. If
   * the filter requires all instances prior to filtering, output()
   * may now be called to retrieve the filtered instances. Any
   * subsequent instances filtered should be filtered based on setting
   * obtained from the first batch (unless the inputFormat has been
   * re-assigned or new options have been set). This default
   * implementation assumes all instance processing occurs during
   * inputFormat() and input().
   *
   * @return true if there are instances pending output
   * @exception NullPointerException if no input structure has been defined,
   * @exception Exception if there was a problem finishing the batch.
   */
  public boolean batchFinished() throws Exception {

    if (getInputFormat() == null) {
      throw new NullPointerException("No input instance format defined");
    }
    m_NewBatch = true;
    return (numPendingOutput() != 0);
  }

  /**
   * Main method for testing this class.
   *
   * @param argv should contain arguments to the filter: use -h for help
   */
  public static void main(String [] argv) {
    
    try {
      if (Utils.getFlag('b', argv)) {
	Filter.batchFilterFile(new KEAPhraseFilter(), argv);
      } else {
	Filter.filterFile(new KEAPhraseFilter(), argv);
      }
    } catch (Exception ex) {
      System.out.println(ex.getMessage());
    }
  }
 
  /** 
   * Converts an instance by removing all non-alphanumeric characters
   * from its string attribute values.
   */
  private void convertInstance(Instance instance) throws Exception {
  
    double[] instVals = new double[instance.numAttributes()];

    for (int i = 0; i < instance.numAttributes(); i++) {
      if (!instance.attribute(i).isString() || 
	  instance.isMissing(i)) {
	instVals[i] = instance.value(i);
      } else {
	if (!m_SelectCols.isInRange(i)) {
	  int index = getOutputFormat().attribute(i).
	    addStringValue(instance.stringValue(i));
	  instVals[i] = (double)index;
	  continue;
	}
	String str = instance.stringValue(i);
	StringBuffer resultStr = new StringBuffer();
	int j = 0;
	boolean phraseStart = true;
	boolean seenNewLine = false;
	boolean haveSeenHyphen = false;
	boolean haveSeenSlash = false;
	while (j < str.length()) {
	  boolean isWord = false;
	  boolean potNumber = false;
	  int startj = j;
	  while (j < str.length()) {
	    char ch = str.charAt(j);
	    if (Character.isLetterOrDigit(ch)) {
	      potNumber = true;
	      if (Character.isLetter(ch)) {
		isWord = true;
	      }
	      j++;
	    } else if ((!m_DisallowInternalPeriods && (ch == '.')) ||
		       (ch == '@') ||
		       (ch == '_') ||
		       (ch == '&') ||
		       (ch == '/') ||
		       (ch == '-')) {
	      if ((j > 0) && (j  + 1 < str.length()) &&
		  Character.isLetterOrDigit(str.charAt(j - 1)) &&
		  Character.isLetterOrDigit(str.charAt(j + 1))) {
		j++;
	      } else {
		break;
	      }
	    } else if (ch == '\'') {
	      if ((j > 0) &&
		  Character.isLetterOrDigit(str.charAt(j - 1))) {
		j++;
	      } else {
		break;
	      }
	    } else {
	      break;
	    }
	  }
	  if (isWord == true) {
	    if (!phraseStart) {
	      if (haveSeenHyphen) {
		resultStr.append('-');
	      } else if (haveSeenSlash) {
		resultStr.append('/');
	      } else {
		resultStr.append(' ');
	      }
	    }
	    resultStr.append(str.substring(startj, j));
	    if (j == str.length()) {
	      break;
	    }
	    phraseStart = false;
	    seenNewLine = false;
	    haveSeenHyphen = false;
	    haveSeenSlash = false;
	    if (Character.isWhitespace(str.charAt(j))) {
	      if (str.charAt(j) == '\n') {
		seenNewLine = true;
	      } 
	    } else if (str.charAt(j) == '-') {
	      haveSeenHyphen = true;
	    } else if (str.charAt(j) == '/') {
	      haveSeenSlash = true;
	    } else {
	      phraseStart = true;
	      resultStr.append('\n');
	    }
	    j++;
	  } else if (j == str.length()) {
	    break;
	  } else if (str.charAt(j) == '\n') {
	    if (seenNewLine) {
	      if (phraseStart == false) {
		resultStr.append('\n');
		phraseStart = true;
	      }
	    } else if (potNumber) {
	      if (phraseStart == false) {
		phraseStart = true;
		resultStr.append('\n');
	      }
	    }
	    seenNewLine = true;
	    j++;
	  } else if (Character.isWhitespace(str.charAt(j))) {
	    if (potNumber) {
	      if (phraseStart == false) {
		phraseStart = true;
		resultStr.append('\n');
	      }
	    }
	    j++;
	  } else {
	    if (phraseStart == false) {
	      resultStr.append('\n');
	      phraseStart = true;
	    }
	    j++;
	  }
	}
	int index = getOutputFormat().attribute(i).
	  addStringValue(resultStr.toString());
	instVals[i] = (double)index;
      }
    }
    Instance inst = new Instance(instance.weight(), instVals);
    inst.setDataset(getOutputFormat());
    push(inst);
  }

  /**
   * Returns the tip text for this property
   *
   * @return tip text for this property suitable for
   * displaying in the explorer/experimenter gui
   */
  public String invertSelectionTipText() {

    return "If set to false, the specified attributes will be processed;"
      + " If set to true, specified attributes won't be processed.";
  }

  /**
   * Get whether the supplied columns are to be processed
   *
   * @return true if the supplied columns won't be processed
   */
  public boolean getInvertSelection() {

    return m_SelectCols.getInvert();
  }

  /**
   * Set whether selected columns should be processed. If true the 
   * selected columns won't be processed.
   *
   * @param invert the new invert setting
   */
  public void setInvertSelection(boolean invert) {

    m_SelectCols.setInvert(invert);
  }

  /**
   * Returns the tip text for this property
   *
   * @return tip text for this property suitable for
   * displaying in the explorer/experimenter gui
   */
  public String disallowInternalPeriodsTipText() {

    return "If set to false, internal periods are allowed.";
  }

  /**
   * Get whether the supplied columns are to be processed
   *
   * @return true if the supplied columns won't be processed
   */
  public boolean getDisallowInternalPeriods() {

    return m_DisallowInternalPeriods;
  }

  /**
   * Set whether selected columns should be processed. If true the 
   * selected columns won't be processed.
   *
   * @param invert the new invert setting
   */
  public void setDisallowInternalPeriods(boolean disallow) {

    m_DisallowInternalPeriods = disallow;
  }

  /**
   * Returns the tip text for this property
   *
   * @return tip text for this property suitable for
   * displaying in the explorer/experimenter gui
   */
  public String attributeIndicesTipText() {

    return "Specify range of attributes to act on."
      + " This is a comma separated list of attribute indices, with"
      + " \"first\" and \"last\" valid values. Specify an inclusive"
      + " range with \"-\". E.g: \"first-3,5,6-10,last\".";
  }

  /**
   * Get the current range selection.
   *
   * @return a string containing a comma separated list of ranges
   */
  public String getAttributeIndices() {

    return m_SelectCols.getRanges();
  }

  /**
   * Set which attributes are to be processed
   *
   * @param rangeList a string representing the list of attributes.  Since
   * the string will typically come from a user, attributes are indexed from
   * 1. <br>
   * eg: first-3,5,6-last
   */
  public void setAttributeIndices(String rangeList) {

    m_SelectCols.setRanges(rangeList);
  }

  /**
   * Set which attributes are to be processed
   *
   * @param attributes an array containing indexes of attributes to select.
   * Since the array will typically come from a program, attributes are indexed
   * from 0.
   */
  public void setAttributeIndicesArray(int [] attributes) {
    
    setAttributeIndices(Range.indicesToRangeList(attributes));
  }
}
