package de.lcag.common;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.map.ListOrderedMap;
import org.apache.commons.collections4.set.ListOrderedSet;
import org.apache.commons.io.FileUtils;
import org.apache.commons.text.StringEscapeUtils;

import routines.LcagDateTime;
import routines.LcagLogger;
import routines.LcagStringTools;

public abstract class AbstractTable implements Table {
	protected static final String PARAMETER_STYLE = ";style=";
	private static final String HTML_TABLE_STYLE = "text-align:left;border:1px solid black;border-collapse:collapse;";
	private static final Pattern PATTERN_FIND_ALL_STARTINGWITH_HTML_UNTIL_NEXT_BLANC = Pattern
			.compile("#HTML#(.*?)([\\n\\s+\\n]|$)");

	protected final LcagLogger log = LcagLogger.getLogger(this);

	protected Object defaultValue = null;
	protected String defaultString = String.valueOf((String) defaultValue);
	protected SimpleDateFormat defaultDateFormat = null;
	protected String separator = "; ";
	protected char encloseBy = '"';
	protected String eol = "\n";

	private int noRows = 0;
	private int firstRowNo = 0;
	private int maxRows = -1;

	private PositionOrderedMap<String, TableColumn> rowMap = new PositionOrderedMap<String, TableColumn>();
	private Set<String> keyColumns = new ListOrderedSet<>();
	private List<String> wantedColumns = new ArrayList<>();
	private List<String> excludedColumns = new ArrayList<>();
	protected Map<String, Object> options = new HashMap<>();
	private KeyColumn keyColumn = null;

	// List of columns: ALso used to ensure correct sequence (!). Maps UPPER Case
	// column IDs to User given labels
	protected ListOrderedMap<String, String> colNames = new ListOrderedMap<>();

	public class ListColumn implements TableColumn {
		List<Object> data = new ArrayList<>();
		private Object defaultValue = null;
		private String style = "";

		public ListColumn() {
		}

		public ListColumn(Object theDefaultValue) {
			this.defaultValue = theDefaultValue;
		}

		public ListColumn(ListColumn pSource) {
			this.defaultValue = pSource.defaultValue;
			this.data = new ArrayList<>(pSource.data);
		}

		public void addValue(Object v) {
			data.add(v);
		}

		public Object getValue(int i) {
			Object result = null;
			int n = data.size();

			i -= AbstractTable.this.firstRowNo;

			if (i < n) {
				result = data.get(i);

				if (result == null)
					result = this.defaultValue;
			}

			return result;
		}

		public String aggregate(String pSeparator) {
			String result = LcagStringTools.joinIgnoreNull(pSeparator, this.data);

			return result;
		}

		public void setValue(int rowNo, Object v) {
			int n = this.data.size();

			rowNo -= AbstractTable.this.firstRowNo;

			if (rowNo < n) {
				data.set(rowNo, v);
			}
			if (n == rowNo) {
				data.add(v);
			}
		}

		public String asCSV(String pSeparator) {
			String result = LcagStringTools.joinIgnoreNull(pSeparator, this.data);

			return result;
		}

		public int findRow(Object v) {
			int result = data.indexOf(v) + firstRowNo;

			return result;
		}

		public String getStyle() {
			return this.style;
		}

		public void setStyle(String pStyle) {
			this.style = pStyle;
		}

		@Override
		public int length() {
			return this.data.size();
		}

		@Override
		public int shift() {
			// TODO Auto-generated method stub
			return -1;
		}

		@Override
		public void init(TableColumn clientColumn) {
			int len = clientColumn.length();
			data = new ArrayList<>();

			for (int i = 0; i < len; i++) {
				Object value = clientColumn.getValue(i);

				this.data.add(value);
			}
		}

		@Override
		public void init(Object pValue) {
			int len = this.length();
			data = new ArrayList<>();

			for (int i = 0; i < len; i++) {
				this.data.add(pValue);
			}
		}

		@Override
		public int getPosition() {
			return AbstractTable.this.getColumnPosition(this);
		}

		@Override
		public Table getTable() {
			return AbstractTable.this;
		}
	}

	public class RowNumberColumn extends ListColumn {
		public RowNumberColumn(AbstractTable pTable) {
			int noRows = pTable.lenght();

			for (int i = 0; i < noRows; i++) {
				this.addValue(i);
			}
		}
	}

	public class ReferenceColumn extends ListColumn {
		private Map<String, Integer> string2RowNum = new LinkedHashMap<>();

		@Override
		public void addValue(Object v) {
			int n = string2RowNum.size();
			String value = v.toString();
			Integer existingRowNo = string2RowNum.get(value);

			if (existingRowNo == null) {
				existingRowNo = this.length();
				string2RowNum.put(value, existingRowNo);
				super.addValue(v);
			}
		}

		@Override
		public void setValue(int rowNo, Object v) {
			int n = this.string2RowNum.size();
			String value = v.toString();
			Integer existingRowNo = string2RowNum.get(value);

			rowNo -= AbstractTable.this.firstRowNo;

			if (existingRowNo == null) {
				if (rowNo < n) {
					this.string2RowNum.put(value, rowNo);
					super.setValue(rowNo, v);
				}

				if (n < rowNo + firstRowNo) {
					this.addValue(v);
				}
			}
		}

		public Integer getRowNo(String pKey) {
			Integer result = null;

			result = this.string2RowNum.get(pKey);

			if (result != null)
				result += firstRowNo;

			return result;
		}
	}

	public class KeyColumn extends ReferenceColumn {
	}

	public AbstractTable() {
		this.colNames = new ListOrderedMap<>();
		this.rowMap = new PositionOrderedMap<String, TableColumn>();

		this.noRows = 0;
	}

	public AbstractTable(int pStartRow) {
		this();

		this.firstRowNo = pStartRow;
	}

	@Override
	public void init() {
	}

	@Override
	public int newRow() {
		int result = noRows;
		noRows += 1;

		return result;
	}

	@Override
	public int lenght() {
		return noRows;
	}

	@Override
	public int getNoColumns() {
		return this.colNames.size();
	}

	public LinkedHashMap<String, Object> getRow(int i) {
		LinkedHashMap<String, Object> result = new LinkedHashMap<>();

		for (String colName : this.getColumnNames()) {
			Object value = this.getCellValue(i, colName);

			result.put(colName, value);
		}

		return result;
	}

	@Override
	public Integer getRowByKey(String pKey) {
		Integer result = this.keyColumn.getRowNo(pKey);

		return result;
	}

	public TableColumn cloneColumn(String pName) {
		int n = pName.indexOf('.'); // Check if it starts with a table name
		String colId = n >= 0 ? pName.substring(n + 1).toUpperCase() : getColumnNameByLabel(pName);
		TableColumn srcCol = this.getColumnByLabel(colId);
		TableColumn result = new ListColumn(srcCol);

		return result;
	}

	@Override
	public void addColumns(String... aNameList) {
		for (String aName : aNameList) {
			this.addColumn(aName);
		}
	}

	public static Path resolveFilePath(String aFilePath, Map<String, Object> pParam) {
		String filePath = LcagStringTools.resolveVariables(aFilePath, pParam);

		filePath = LcagStringTools.resolveEnvironmentVariables(filePath);

		return Path.of(filePath);
	}

	/**
	 * Sets the cell value for the given row number and column name to the given
	 * value. Like {@link #setCellValue(int, String, Object)}, but creates the
	 * column if it does not yet exist.
	 * 
	 * @param pRowNo
	 * @param pColumnLabel
	 * @param pValue
	 */
	public void addCellValue(int pRowNo, String pColumnLabel, Object pValue) {
		this.addColumn(pColumnLabel);
		this.setCellValue(pRowNo, pColumnLabel, pValue);
	}

	public void addCellValues(Map<String, Object> pValues) {
		if (pValues == null || pValues.isEmpty())
			return;

		int rowNo = this.newRow();

		for (String key : pValues.keySet()) {
			Object value = pValues.get(key);

			this.setCellValue(rowNo, key, value);
		}
	}

	public void setCellValue(int row, int pColumnNumber, Object v) {
		String colName = this.getColumnNameByPosition(pColumnNumber);

		this.setCellValue(row, colName, v);
	}

	@Override
	public void setCellValue(int row, String colName, Object v) {
		TableColumn col = getColumnByLabel(colName);

		col.setValue(row, v);

		if (keyColumn != null && this.isPartOfKey(colName)) {
			Object keyValue = keyColumn.getValue(row);

			if (keyValue == null) {
				Object key = this.getKeyValue(row);

				if (key != null)
					keyColumn.setValue(row, key);
			}
		}
	}

	@Override
	public void setCellValues(int pMyRow, Table pSrcTable, Integer pSrcRow) {
		for (String colName : pSrcTable.getColumnNames()) {
			if (this.colNames.get(colName) != null) {
				Object value = pSrcTable.getCellValue(pSrcRow, colName);

				this.setCellValue(pMyRow, colName, value);
			}
		}
	}

	public void setCellValues(int pMyRow, Map<String, Object> pFieldValues) {
		for (String colName : pFieldValues.keySet()) {
			if (this.colNames.get(colName) != null) {
				Object value = pFieldValues.get(colName);

				this.setCellValue(pMyRow, colName, value);
			}
		}
	}

	public void setMaxRows(int pMaxRows) {
		maxRows = pMaxRows;
	}

	@Override
	public <V> V getCellValue(int row, String colLabel) {
		TableColumn col = getColumnByLabel(colLabel);
		V result = null;

		if (col == null) {
			// FIXME: Shall throw a TableColumnException (bigger impact)
			log.fatal("Column '%s' does not exist in %s", colLabel, this.getName());
		} else {
			result = col.getValue(row);
		}

		return result;
	}

	@Override
	public String asCSV(String pColName, String pSeparator) {
		String result = "";
		TableColumn col = this.getColumnByLabel(getColumnNameByLabel(pColName));

		if (col != null) {
			result = col.asCSV(pSeparator);
		}

		return result;
	}

	@Override
	public int findRow(String colName, Object v) {
		int result = -1;

		TableColumn col = this.getColumnByLabel(getColumnNameByLabel(colName));

		if (col != null) {
			result = col.findRow(v);
		}

		return result;
	}

	@Override
	public void joinTable(Table pTargetTable, Table pTableToJoin, Map<String, String> pJoinFieldMap) {
		Set<String> joinTableColumns = pJoinFieldMap.keySet();
		Collection<String> joinColLabelList = pJoinFieldMap.values();
		Collection<String> myColumnLabels = this.getColumnLabels();

		pTargetTable.init();

		pTargetTable.addColumns(myColumnLabels);

		// Check precondition: All given columns must exists:
		for (String colLabel : joinColLabelList) {
			String colName = getColumnNameByLabel(colLabel);

			if (this.colNames.get(colName) == null) {
				log.debug("Column '%s' does not exist in this Table => empty join result", colLabel);
				return;
			}
		}

		// Add column names of table to join that are not yet known in the traget table:
		for (String colLabel : pTableToJoin.getColumnLabels()) {
			String colName = getColumnNameByLabel(colLabel);

			if (!joinTableColumns.contains(colLabel)) {
				pTargetTable.addColumn(colLabel);
			}

		}

		int joinTableLenght = pTableToJoin.lenght();

		// Join the data
		for (int i = firstRowNo; i < joinTableLenght; i++) {
			List<Integer> matchingRowNumbers = this.selectRows(pTableToJoin, pJoinFieldMap, i);

			for (Integer j : matchingRowNumbers) {
				int newRow = pTargetTable.newRow();

				pTargetTable.setCellValues(newRow, this, j);
				pTargetTable.setCellValues(newRow, pTableToJoin, i);

			}
		}

	}

	public HashTable select(String pTableName) {
		return (HashTable) Table.selectFromTable(this, pTableName);
	}

	/**
	 * Select all rows of this table where ALL field values listed in the map match
	 * the data in the given foreign row number.
	 * 
	 * @param pTable        - Table to select the given row number
	 * @param pJoinFieldMap - Field mapping of foreign field names to this field
	 *                      names. Data of all fields must match
	 * @param pTableRow     - Data row number in foreign table
	 * @return - List of row number in this table that match
	 */
	private List<Integer> selectRows(Table pTable, Map<String, String> pJoinFieldMap, int pTableRow) {
		List<Integer> result = new ArrayList<>();
		Set<String> colLabelList = pJoinFieldMap.keySet();

		for (int i = firstRowNo; i < this.lenght(); i++) {
			boolean matches = true;

			for (String otherColLabel : colLabelList) {
				String myColLabel = pJoinFieldMap.get(otherColLabel);

				Object otherValue = pTable.getCellValue(pTableRow, otherColLabel);
				Object myValue = this.getCellValue(i, myColLabel);

				if (!Objects.deepEquals(myValue, otherValue)) {
					matches = false;
					break;
				}
			}

			if (matches)
				result.add(i);
		}

		return result;
	}

	public void addColumnAtPosition(String pStyledColumnLabel, TableColumn pColumn, int pColumnPosition) {
		String[] col = pStyledColumnLabel.split(PARAMETER_STYLE);
		String columnLabel = col[0];
		String colUniqueName = getColumnNameByLabel(columnLabel);

		if (pColumnPosition < colNames.size())
			shiftColumns(pColumnPosition);

		colNames.put(pColumnPosition, colUniqueName, columnLabel);
		rowMap.put(pColumnPosition, colUniqueName, pColumn);
	}

	public void addColumnAsFirstOne(String pStyledColumnLabel, TableColumn pColumn) {
		this.addColumnAtPosition(pStyledColumnLabel, pColumn, 0);
	}

	public int addColumn(String pStyledColumnLabel, TableColumn pColumn, String pAfterColumnNamed) {
		int colPosResult = -1;
		String[] col = pStyledColumnLabel.split(PARAMETER_STYLE);
		String columnLabel = col[0];
		String colUniqueName = getColumnNameByLabel(columnLabel);

		MappingTable mapping = getOption(OPTION_USE_MAPPING);

		if (mapping != null) {
			columnLabel = mapping.getColumnLableFor(columnLabel);
			colUniqueName = getColumnNameByLabel(columnLabel);
		}

		if (!colNames.containsKey(colUniqueName)) {
			int noItems = pColumn.length();

			if (col.length > 1) {
				pColumn.setStyle(col[1]);
			}

			if (pAfterColumnNamed == null) {
				colNames.put(colUniqueName, columnLabel);
				rowMap.put(colUniqueName, pColumn);
				colPosResult = rowMap.size() - 1;
			} else {
				colPosResult = this.getColumnPositionByName(pAfterColumnNamed) + 1;

				if (colPosResult < colNames.size())
					shiftColumns(colPosResult);
				colNames.put(colPosResult, colUniqueName, columnLabel);
				rowMap.put(colPosResult, colUniqueName, pColumn);
				rowMap.toString();
			}

			if (this.noRows < noItems) {
				this.noRows = noItems;
			}
		}

		return colPosResult;
	}

	public String getColumnNameByPosition(int pPosition) {
		return this.rowMap.getKeyByPosition(pPosition);
	}

	public int shiftColumns(int pStartPosition) {
		return this.rowMap.shiftAllPositions(pStartPosition);
	}

	@Override
	public int getFirstRowNo() {
		return this.firstRowNo;
	}

	public String getLastColumnName() {
		return this.rowMap.lastKey();
	}

	public int getLastColumnPosition() {
		String colName = this.getLastColumnName();
		return this.getColumnPositionByName(colName);
	}

	public int addColumn(String pStyledColumnLabel, TableColumn pColumn) {
		return this.addColumn(pStyledColumnLabel, pColumn, null);
	}

	@Override
	public TableColumn addColumn(String pStyledColumnLabel) {
		ListColumn newCol = new ListColumn();

		this.addColumn(pStyledColumnLabel, newCol);

		return newCol;
	}

	@Override
	public TableColumn getColumnByName(String colName) {
		TableColumn result = this.rowMap.get(colName);

		return result;
	}

	@Override
	public TableColumn getColumnByLabel(String pFormattedColName) {
		String colName = pFormattedColName.split(PARAMETER_STYLE)[0];
		TableColumn result = this.rowMap.get(getColumnNameByLabel(colName));
		MappingTable mapping = getOption(OPTION_USE_MAPPING);

		if (mapping != null) {
			String colLabel = mapping.getColumnLableFor(colName);
			result = this.rowMap.get(getColumnNameByLabel(colLabel));
		}

		return result;
	}

	@Override
	public void addColumns(Collection<String> pNameList) {
		for (String aName : pNameList) {
			this.addColumn(aName);
		}
	}

	public String aggregateColumn(String pColName, String pSeparator) {
		String result = "";
		ListColumn col = (ListColumn) this.getColumnByLabel(getColumnNameByLabel(pColName));

		if (col != null) {
			result = col.aggregate(pSeparator);
		}

		return result;
	}

	public String asCsvData(TableColumn pOutputOrdering) {
		String result = "";
		boolean numberRows = this.optionIsSet(OPTION_NUMBER_ROWS);

		for (int i = firstRowNo; i < this.lenght(); i++) {
			List<String> data = new ArrayList<>();
			int rowNo = i;

			if (pOutputOrdering != null) {
				rowNo = (int) pOutputOrdering.getValue(i);
			}

			if (numberRows)
				data.add(String.format("%05d", rowNo));

			for (String colName : this.getColumnNames()) {
				TableColumn col = this.getColumnByName(colName);
				Object colValue = col.getValue(rowNo);

				if (colValue == null) {
					colValue = this.defaultString;
				} else if (colValue instanceof Date) {
					colValue = LcagDateTime.formatDate(this.defaultDateFormat, colValue);
				} else if (colValue instanceof List<?>) {
					List<?> valueList = (List<?>) colValue;
					List<String> sList = new ArrayList<>();

					for (Object o : valueList) {
						sList.add(o.toString());
					}

					colValue = String.join(this.separator, sList);
				}

				data.add(colValue.toString());
			}

			result += String.join(this.separator, data) + this.eol;
		}

		return result;
	}

	@Override
	public String asCsvData() {
		return this.asCsvData(null);
	}

	public String asCSV(TableColumn pOutputOrdering) {
		final String separator = this.getSeparator();
		boolean addRowNumber = this.optionIsSet(OPTION_NUMBER_ROWS);
		String result = "";

		if (addRowNumber) {
			result = ROW_NUMBER_COLUMN + separator;
		}

		result += String.join(separator, this.getColumnNames());
		result += this.eol;
		result += this.asCsvData(pOutputOrdering);
		return result;
	}

	public String asHTML(TableColumn pOutputOrdering) {
		String style = HTML_TABLE_STYLE;
		LinkedHashSet<String> colNames = new LinkedHashSet<>();

		String result = String.format(
				"<!DOCTYPE html>\r\n" + "<html lang=\"en\">\r\n" + "<head>\r\n" + "    <meta charset=\"UTF-8\">\r\n"
						+ "    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\r\n"
						+ "    <title>%s</title>\r\n" + "</head>\r\n",
				this.getName());
		Set<String> columnNames = this.getColumnNames();
		boolean addRowNumber = this.optionIsSet(OPTION_NUMBER_ROWS);

		result += String.format("<table style=\"%s;width: 100%%\"><tr>", style) + this.eol;

		if (addRowNumber) {
			result += String.format("\t\t<th style=\"%s\">%s</th>\n", style, ROW_NUMBER_COLUMN);
		}

		colNames.addAll(this.getColumnNames());

		// generate table header
		for (String colName : colNames) {
			TableColumn col = this.getColumnByName(colName);
			colName = StringEscapeUtils.escapeHtml4(this.colNames.get(colName));

			String colStyle = col.getStyle();
			result += String.format("\t\t<th style=\"%s\">%s</th>\n", style + colStyle, colName);
		}
		result += "</tr>" + this.eol;

		// generate table data
		for (int i = firstRowNo; i < this.lenght(); i++) {
			List<String> data = new ArrayList<>();
			int rowNo = i;

			if (pOutputOrdering != null) {
				rowNo = (int) pOutputOrdering.getValue(i);
			}

			if (addRowNumber) {
				data.add(String.format("%05d", rowNo));
			}
			for (String colName : columnNames) {
				TableColumn col = this.getColumnByName(colName);

				if (col == null)
					continue;

				Object colValue = col.getValue(rowNo);
				String colStyle = col.getStyle();

				if (colValue == null) {
					colValue = this.defaultString;
				} else if (colValue instanceof Date) {
					colValue = LcagDateTime.formatDate(this.defaultDateFormat, colValue);
				}

				String valueString = String.valueOf(colValue);

				if (valueString.startsWith("<html>") && valueString.endsWith("</html>")) {
					valueString = valueString.replace("<html>", "");
					valueString = valueString.replace("</html>", "");
				} else {
					valueString = StringEscapeUtils.escapeHtml4(valueString);
					valueString = convertHtmlInHtml(valueString);
				}
				valueString = String.format("\t\t<td style=\"%s\">%s</td>", style, valueString);

				data.add(valueString);
			}

			result += "\t<tr>" + this.eol + String.join(this.eol, data) + "</tr>" + this.eol;
		}

		result += "</tr></table>" + this.eol;

		return result;

	}

	private String convertHtmlInHtml(String valueString) {
		if (valueString.contains("#HTML#")) {
			StringBuilder result = new StringBuilder();
			Matcher matcher = PATTERN_FIND_ALL_STARTINGWITH_HTML_UNTIL_NEXT_BLANC.matcher(valueString);

			int lastEnd = 0;
			while (matcher.find()) {
				String startString = valueString.substring(lastEnd, matcher.start());
				result.append(startString);
				final String matchingString = matcher.group(1);
				String htmlContent = matchingString.trim();
				result.append("<" + htmlContent + ">");
				lastEnd = matcher.end();
			}

			result.append(valueString.substring(lastEnd));

			valueString = result.toString();
		}
		return valueString;
	}

	@Override
	public String asHTML() {
		return asHTML(null);
	}

	public void append(Table pSrcTable) {
		int noRows = pSrcTable.lenght();
		Collection<String> colNames = pSrcTable.getColumnNames();

		for (int rowNo = pSrcTable.getFirstRowNo(); rowNo < noRows; rowNo++) {
			int newRowNo = this.newRow();

			for (String colName : colNames) {
				Object value = pSrcTable.getCellValue(rowNo, colName);

				if (rowNo == 0 && this.colNames.get(colName) == null) {
					this.addColumn(colName);
				}

				this.setCellValue(newRowNo, colName, value);
			}
		}
	}

	@Override
	public Table joinTable(Table pTable, Map<String, String> pJoinFieldMap) {
		HashTable result = new HashTable();

		this.joinTable(result, pTable, pJoinFieldMap);
		return result;
	}

	@Override
	public String toString() {
		String result = String.join(this.separator, this.getColumnLabels()) + this.eol;

		result += this.asCsvData();

		return result;
	}

	@Override
	public String asCSV() {
		final String separator = this.getSeparator();
		boolean addRowNumber = this.optionIsSet(OPTION_NUMBER_ROWS);
		String result = "";

		if (addRowNumber) {
			result = ROW_NUMBER_COLUMN + separator;
		}

		result += String.join(separator, this.getColumnNames());
		result += this.eol;
		result += this.asCsvData();
		return result;
	}

	public String getEol() {
		return eol;
	}

	public void setEol(String eol) {
		this.eol = eol;
	}

	public String getSeparator() {
		return separator;
	}

	public void setSeparator(String separator) {
		this.separator = separator;
	}

	public SimpleDateFormat getDefaultDateFormat() {
		return defaultDateFormat;
	}

	public void setDefaultValue(Object defaultValue) {
		this.defaultValue = defaultValue;
		this.defaultString = String.valueOf((String) defaultValue);
	}

	public void setDefaultDateFormat(SimpleDateFormat defaultDateFormat) {
		this.defaultDateFormat = defaultDateFormat;
	}

	public void setWantedColumns(List<String> pWantedColumns) {
		for (String s : pWantedColumns) {
			this.wantedColumns.add(s);
		}
	}

	public void setExcludedColumns(List<String> pExcludedColumns) {
		for (String s : pExcludedColumns) {
			this.excludedColumns.add(s);
		}
	}

	protected void setKeyColumns(List<String> pKeyColumns) {
		this.keyColumn = new KeyColumn();

		for (String s : pKeyColumns) {
			this.keyColumns.add(s);
		}
	}

	public boolean isPartOfKey(String pKeyPart) {
		return this.keyColumns.contains(pKeyPart);
	}

	public void buildKey() {
		int noRows = this.lenght();

		for (int n = firstRowNo; n < noRows + firstRowNo; n++) {
			this.getKeyValue(n);
		}
	}

	@Override
	public String getKeyValue(int pRow) {
		String result = null;

		if (this.keyColumn != null) {
			// get already cached assembled key
			result = (String) this.keyColumn.getValue(pRow);
		}

		if (result == null) {
			// calculate key
			result = setUniqueKeyValue(pRow);
		}
		return result;
	}

	private String calculateKeyFromColumns(int pRow, Set<String> pColumnNames) {
		String result = null;
		List<Object> keyValues = new ArrayList<>();

		for (String keyPart : pColumnNames) {
			Object v = this.getCellValue(pRow, keyPart);

			if (v == null)
				continue;

			if (v instanceof Date) {
				v = LcagDateTime.DATETIME_FORMAT_ISO8601_UTC.format((Date) v);
			}
			keyValues.add(v);
		}

		result = LcagStringTools.joinIgnoreNull("/", keyValues);

		if (result.length() > 30) {
			// Only use md5 if the String is longer than the MD5 checksum.
			result = DigestUtils.md5Hex(result);
		}

		return result;
	}

	@Override
	public String setKeyValue(int pRow) {
		String resultKeyValue = calculateKeyFromColumns(pRow, this.keyColumns);

		if (resultKeyValue == null) {
			// key columns are all null or the key is not unique => calculate key over all
			// columns
			resultKeyValue = calculateKeyFromColumns(pRow, this.getColumnNames());
		}

		this.keyColumn.setValue(pRow, resultKeyValue);

		return resultKeyValue;

	}

	@Override
	public String setUniqueKeyValue(int pRow) {
		String resultKeyValue = calculateKeyFromColumns(pRow, this.keyColumns);

		if (resultKeyValue == null) {
			// key columns are all null or the key is not unique => calculate key over all
			// columns
			resultKeyValue = calculateKeyFromColumns(pRow, this.getColumnNames());
		}

		// Next check if key is unique. If not: => calculate key over all columns
		Integer existingKeyValue = this.getRowByKey(resultKeyValue);

		if (existingKeyValue != null) {
			resultKeyValue = calculateKeyFromColumns(pRow, this.getColumnNames());
		}

		this.keyColumn.setValue(pRow, resultKeyValue);

		return resultKeyValue;

	}

	protected boolean hasKey() {
		return this.keyColumn != null;
	}

	public void setKey(int pRowNo) {
		String key = this.getKeyValue(pRowNo);

		this.keyColumn.setValue(pRowNo, key);
	}

	public void setOption(String pOption, Object pValue) {
		this.options.put(pOption, pValue);
	}

	public void enableOption(String pOption) {
		this.options.put(pOption, true);
	}

	@SuppressWarnings("unchecked")
	public <T> T getOption(String pOption) {
		return (T) this.options.get(pOption);
	}

	public boolean optionIsSet(String pOption) {
		return this.options.containsKey(pOption);
	}

	public String getColumnNameByLabel(String pStyledColName) {
		final String[] split = pStyledColName.split(PARAMETER_STYLE);
		String colName = split[0];

		return colName.toUpperCase();
	}

	@Override
	public boolean hasColumn(String pName) {
		String columnName = this.colNames.get(getColumnNameByLabel(pName));

		return columnName != null;
	}

	@Override
	public final Set<String> getColumnNames() {
		return this.colNames.keySet();
	}

	@Override
	public int getColumnPositionByName(String pName) {
		TableColumn tableColumn = this.rowMap.get(pName);
		return this.rowMap.getPositionOfValue(tableColumn);
	}

	public int getColumnPosition(TableColumn pCol) {
		final Integer positionOfValue = this.rowMap.getPositionOfValue(pCol);

		return positionOfValue;
	}

	@Override
	public int getColumnPositionByLabel(String pLabel) {
		String name = getColumnNameByLabel(pLabel);
		return getColumnPositionByName(name);
	}

	@Override
	public final Collection<String> getColumnLabels() {
		return this.colNames.values();
	}

	public Object getDefaultValue() {
		return this.defaultValue;
	}

	public boolean useColumn(String pColumnLabel) {
		boolean result = false;

		MappingTable mapping = getOption(OPTION_USE_MAPPING);

		if (mapping != null) {
			pColumnLabel = mapping.getColumnLableFor(pColumnLabel);
		}

		// don't use unmapped columns in case a mapping is given:
		if (pColumnLabel != null) {
			String colName = getColumnNameByLabel(pColumnLabel);

			result = this.keyColumns.contains(pColumnLabel);

			if (!result) {
				if (this.wantedColumns.isEmpty()) {
					result = !this.excludedColumns.contains(pColumnLabel);
				} else {
					result = this.wantedColumns.contains(pColumnLabel);
				}
			}
		}
		return result;
	}

	private void compareStructureTo(TableComparisonResult pResult, Table pCmpTable) {
		Collection<String> myColumumnNames = this.getColumnNames();
		Collection<String> cmpColumumnNames = pCmpTable.getColumnNames();

		Collection<String> commonColumnNames = CollectionUtils.intersection(myColumumnNames, cmpColumumnNames);
		pResult.setCommonColumns(commonColumnNames);

		Collection<String> myPrivateColumnNames = CollectionUtils.subtract(myColumumnNames, commonColumnNames);
		pResult.setColumnsToDelete(myPrivateColumnNames);

		Collection<String> cmpPrivateColumnNames = CollectionUtils.subtract(cmpColumumnNames, commonColumnNames);
		pResult.setColumnsToAdd(cmpPrivateColumnNames);
	}

	public TableComparisonResult compareTo(AbstractTable pCmpTable) {
		TableComparisonResult result = new TableComparisonResult(this, pCmpTable);
		List<Integer> cmpRowsDone = new ArrayList<>();
		this.compareStructureTo(result, pCmpTable);
		Collection<String> commonColumns = result.getCommonColumns();
		int totalNumberOfComparedCells = 0;
		int totalNumberOfDifferentCells = 0;

		pCmpTable.buildKey();
		this.buildKey();

		totalNumberOfComparedCells = (result.getColumnsToDeleteFromMaster().size()
				+ result.getColumnsToAddToMaster().size()) * this.lenght();
		totalNumberOfComparedCells += (result.getRowsToDeleteFromMaster().size() * this.getNoColumns()
				+ result.getRowsToAddFromClient().size() * pCmpTable.getNoColumns());
		totalNumberOfDifferentCells = totalNumberOfComparedCells;

		// Phase 1: Compare my rows to those in cmp table
		int lenght = this.lenght();
		int maxRowsToCompare = (maxRows > 0 && maxRows < lenght) ? maxRows : lenght;

		for (int n = this.firstRowNo; n < maxRowsToCompare + firstRowNo; n++) {
			boolean rowsAreEqual = true;
			int perCent = (n * 100) / lenght;

			String thisKey = this.getKeyValue(n);
			Integer cmpRowNo = pCmpTable.getRowByKey(thisKey);

			if (perCent % 5 == 0)
				log.info("compare: %5d/%d rows done = %2d%%", n, lenght, perCent);

			if (cmpRowNo == null) {
				// row does not exist in compare table => add delete instruction in master
				result.addRowToDelete(n);
			} else {
				cmpRowsDone.add(cmpRowNo);
				totalNumberOfComparedCells += result.getCommonColumns().size();

				for (String colName : commonColumns) {
					Object thisValue = this.getCellValue(n, colName);
					Object cmpValue = pCmpTable.getCellValue(cmpRowNo, colName);

					if (thisValue == null && cmpValue == null) {
						continue;
					} else if (thisValue != null && cmpValue != null) {
						if (thisValue instanceof String && !(cmpValue instanceof String)) {
							cmpValue = cmpValue.toString();
						} else if (cmpValue instanceof String && !(thisValue instanceof String)) {
							thisValue = thisValue.toString();
						}
					} 
					
					if (!Objects.deepEquals(thisValue, cmpValue)) {
						result.addCellDifference(thisKey, n, colName, cmpRowNo, colName);
						totalNumberOfDifferentCells += 1;
						rowsAreEqual = false;
					}
				}
			}
		}

		// Phase 2: Protocol those rows from cmp table that do not exist in my table
		lenght = pCmpTable.lenght();
		maxRowsToCompare = (maxRows > 0 && maxRows < lenght) ? maxRows : lenght;

		for (int n = firstRowNo; n < maxRowsToCompare; n++) {
			if (!cmpRowsDone.contains(n)) {
				result.addRowToAdd(n);
			}
		}

		result.setTotalNumberOfComparedCells(totalNumberOfComparedCells);
		result.setTotalNumberOfDifferentCells(totalNumberOfDifferentCells);
		return result;
	}

	private int partition(TableColumn pResultRowNumbers, TableColumn pColumnToSort, int begin, int end) {
		String pivot = (String) pColumnToSort.getValue(end);
		int i = (begin - 1);

		for (int j = begin; j < end; j++) {
			String cmpValue = (String) pColumnToSort.getValue(j);
			int cmpResult = cmpValue.compareTo(pivot);

			if (cmpResult < 0) {
				i++;

				_quicksortSwapValues(pResultRowNumbers, pColumnToSort, i, j);
			}
		}

		_quicksortSwapValues(pResultRowNumbers, pColumnToSort, i + 1, end);

		return i + 1;
	}

	private void _quicksortSwapValues(TableColumn pResultRowNumbers, TableColumn pColumnToSort, int i, int j) {
		int iRow = (int) pResultRowNumbers.getValue(i);
		int jRow = (int) pResultRowNumbers.getValue(j);
		String iValue = (String) pColumnToSort.getValue(i);
		String jValue = (String) pColumnToSort.getValue(j);

		pResultRowNumbers.setValue(i, jRow);
		pResultRowNumbers.setValue(j, iRow);
		pColumnToSort.setValue(i, jValue);
		pColumnToSort.setValue(j, iValue);
	}

	private void quickSort(TableColumn pResultRowNumbers, TableColumn pColumnToSort, int begin, int end) {
		if (begin < end) {
			int partitionIndex = partition(pResultRowNumbers, pColumnToSort, begin, end);

			quickSort(pResultRowNumbers, pColumnToSort, begin, partitionIndex - 1);
			quickSort(pResultRowNumbers, pColumnToSort, partitionIndex + 1, end);
		}
	}

	/**
	 * Use this function as a helper to sort the table based on multiple columns.
	 * 
	 * It concats values from multiple columns as strings and return the result as a
	 * new ListColum. The function does not add the new ListColumn to the table.
	 * 
	 * @param pColumnsToConcat
	 * @return
	 */
	public TableColumn concatColumns(String... pColumnsToConcat) {
		TableColumn result = new ListColumn();
		List<TableColumn> columnList = new ArrayList<>();

		for (String colName : pColumnsToConcat) {
			TableColumn c = this.getColumnByLabel(colName);

			if (c != null)
				columnList.add(c);
		}

		for (int r = 0; r < this.lenght(); r++) {
			List<Object> valueList = new ArrayList<>();

			for (TableColumn c : columnList) {
				Object val = c.getValue(r);

				valueList.add(val);
			}

			String concatVaue = LcagStringTools.joinAll("#", "(null)", valueList);

			result.addValue(concatVaue);
		}

		return result;
	}

	public TableColumn sort(String... pSortColumns) {
		TableColumn columnToSort = this.concatColumns(pSortColumns);
		TableColumn result = new RowNumberColumn(this);

		quickSort(result, columnToSort, 0, this.lenght() - 1);

		return result;
	}

	static void createDirectoriesInPath(File aFile) {
		File dir = aFile.getParentFile();

		if (dir != null) {
			dir.mkdirs();
		}
	}

	public void saveAs(String pFilePath) throws IOException {
		File aFile = new File(pFilePath);
		String filePath = aFile.getCanonicalPath();
		String text = null;

		if (filePath.toLowerCase().endsWith(".csv")) {
			text = this.toString();
		} else if (filePath.toLowerCase().endsWith(".html")) {
			text = this.asHTML();
		}

		if (text != null) {
			createDirectoriesInPath(aFile);

			FileUtils.writeStringToFile(aFile, text, Charset.forName("UTF-8"));
		}
	}

}