Completed majority of Excel class implementation

core/src/main/java/com/riiablo/excel2/ColumnFormat.java

@@ -0,0 +1,25 @@
+package com.riiablo.excel2;
+
+public class ColumnFormat extends RuntimeException {
+  final String columnText;
+  final int columnIndex;
+
+  ColumnFormat(NumberFormatException t, CharSequence columnText, int columnIndex) {
+    this(t.getMessage(), columnText, columnIndex);
+    initCause(t);
+  }
+
+  ColumnFormat(CharSequence message, CharSequence columnText, int columnIndex) {
+    super(message == null ? null : message.toString());
+    this.columnText = columnText.toString();
+    this.columnIndex = columnIndex;
+  }
+
+  public String columnText() {
+    return columnText;
+  }
+
+  public int columnIndex() {
+    return columnIndex;
+  }
+}

core/src/main/java/com/riiablo/excel2/Excel.java

@@ -12,13 +12,16 @@ import java.util.Iterator;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.commons.lang3.tuple.Triple;
 
 import com.badlogic.gdx.files.FileHandle;
 import com.badlogic.gdx.utils.Array;
-import com.badlogic.gdx.utils.ObjectMap;
+import com.badlogic.gdx.utils.IntMap;
+import com.badlogic.gdx.utils.ObjectIntMap;
 
 import com.riiablo.logger.LogManager;
 import com.riiablo.logger.Logger;
+import com.riiablo.logger.MDC;
 import com.riiablo.util.ClassUtils;
 
 /**
@@ -32,6 +35,16 @@ public abstract class Excel<
 {
   private static final Logger log = LogManager.getLogger(Excel.class);
 
+  /** Forces excels to either have a {@link PrimaryKey} set or be {@link Indexed} */
+  private static final boolean FORCE_PRIMARY_KEY = !true;
+
+  private static final ObjectIntMap EMPTY_OBJECT_INT_MAP = new ObjectIntMap();
+
+  @SuppressWarnings("unchecked") // doesn't store anything
+  static <T> ObjectIntMap<T> emptyMap() {
+    return (ObjectIntMap<T>) EMPTY_OBJECT_INT_MAP;
+  }
+
   /**
    * Root class of an excel entry.
    */
@@ -97,9 +110,10 @@ public abstract class Excel<
   T loadTxt(T excel, FileHandle handle) throws IOException {
     InputStream in = handle.read();
     try {
+      MDC.put("excel", handle.path());
       TxtParser parser = TxtParser.parse(in);
       return loadTxt(excel, parser);
-    } catch (IllegalAccessException|ParseException t) {
+    } catch (Throwable t) {
       log.fatal("Unable to load {} as {}: {}",
           handle,
           excel.getClass().getCanonicalName(),
@@ -107,13 +121,296 @@ public abstract class Excel<
           t);
       return ExceptionUtils.rethrow(t);
     } finally {
+      MDC.remove("excel");
       IOUtils.closeQuietly(in);
     }
   }
 
   static <E extends Entry, S extends Serializer<E>, T extends Excel<E, S>>
-  T loadTxt(T excel, TxtParser parser) throws IOException, IllegalAccessException {
-    throw null;
+  T loadTxt(T excel, TxtParser parser)
+      throws IOException, ParseException, IllegalAccessException
+  {
+    final Class<E> entryClass = excel.entryClass();
+    final boolean indexed = ClassUtils.hasAnnotation(entryClass, Indexed.class);
+
+    final String[] TMP = new String[1];
+    Field primaryKey = null, firstKey = null;
+    Array<Triple<Field, int[], String[]>> columns = new Array<>(true, parser.numColumns(), Triple.class);
+    for (Field field : entryClass.getFields()) {
+      Entry.Column column = field.getAnnotation(Entry.Column.class);
+      if (column == null) {
+        log.warn("{} is not tagged with {}", field, Entry.Column.class.getCanonicalName());
+        continue;
+      }
+
+      PrimaryKey key = field.getAnnotation(PrimaryKey.class);
+      if (key != null) {
+        if (!ArrayUtils.contains(PrimaryKey.SUPPORTED_TYPES, field.getType())) {
+          throw new ParseException(field, "%s must be one of %s",
+              field, Arrays.toString(PrimaryKey.SUPPORTED_TYPES));
+        }
+
+        if (indexed) {
+          // Indexed excels have their primary key assigned automatically based on row index
+          log.warn("{} has {} set to the primary key, but class is tagged with {}",
+              entryClass, field, Indexed.class.getCanonicalName());
+        } else if (primaryKey != null) {
+          // Allow declared field tagged as a primary key to override inherited ones
+          boolean primaryDeclared = ClassUtils.isDeclaredField(entryClass, primaryKey);
+          boolean fieldDeclared = ClassUtils.isDeclaredField(entryClass, field);
+          if (primaryDeclared != fieldDeclared) {
+            if (fieldDeclared) {
+              log.debug("primary key {} changed to {}", primaryKey, field);
+              primaryKey = field;
+            }
+          } else {
+            log.warn("multiple primary keys set within {}: {} and {}",
+                entryClass, primaryKey.getName(), field.getName());
+          }
+        } else {
+          primaryKey = field;
+        }
+      }
+
+      if (firstKey == null) firstKey = field;
+      populateColumnIndexes(columns, parser, column, field, TMP);
+    }
+
+    if (primaryKey == null && !indexed) {
+      if (FORCE_PRIMARY_KEY) {
+        throw new ParseException(entryClass, "%s does not have a %s set!",
+            entryClass, PrimaryKey.class.getCanonicalName());
+      } else {
+        log.warn("{} does not have a {} set! Defaulting to first key: {}",
+            entryClass, PrimaryKey.class.getCanonicalName(), firstKey);
+        primaryKey = firstKey;
+      }
+    }
+
+    // Locate the column index of the primary key
+    // TODO: this operation can be cleaned up, but this is only an identity test
+    int[] primaryKeyColumnIds = null;
+    final Triple<Field, int[], String[]>[] columnTriples = columns.items;
+    for (int i = 0, s = columnTriples.length; i < s; i++) {
+      if (columnTriples[i].getLeft() == primaryKey) {
+        primaryKeyColumnIds = columnTriples[i].getMiddle();
+        break;
+      }
+    }
+
+    int nonzeroIndex = -1;
+    if (!indexed) {
+      for (int i = 0, s = primaryKeyColumnIds.length; i < s; i++) {
+        if (primaryKeyColumnIds[i] >= 0) {
+          nonzeroIndex = i;
+          break;
+        }
+      }
+
+      if (nonzeroIndex == -1) {
+        throw new ParseException(primaryKey,
+            "primary key %s does not have any columns associated with it",
+            primaryKey);
+      }
+    }
+
+    final int primaryKeyColumnId = indexed ? -1 : primaryKeyColumnIds[nonzeroIndex];
+    final Class primaryKeyType = indexed ? null : primaryKey.getType();
+    for (int i = excel.offset(); parser.cacheLine() != -1; i++) {
+      E entry = excel.newEntry();
+      String name = indexed ? null : parser.parseString(primaryKeyColumnId);
+      try {
+        MDC.put("entry", name);
+        inject(excel, entry, name, columns, parser);
+      } finally {
+        MDC.remove("entry");
+      }
+      putIndex(primaryKey, primaryKeyType, i++, indexed, excel, entry);
+    }
+
+    return excel;
+  }
+
+  static <E extends Entry, S extends Serializer<E>, T extends Excel<E, S>>
+  void inject(
+      T excel,
+      E entry,
+      String key,
+      Array<Triple<Field, int[], String[]>> columns,
+      TxtParser parser
+  )
+      throws IllegalAccessException, ParseException
+  {
+    for (Triple<Field, int[], String[]> column : columns) {
+      final Field field = column.getLeft();
+      final int[] columnIds = column.getMiddle();
+      final String[] columnNames = column.getRight();
+      final Class type = field.getType();
+      try {
+        if (type == String.class) {
+          field.set(entry, parser.parseString(columnIds[0]));
+        } else if (type == String[].class) {
+          field.set(entry, parser.parseString(columnIds));
+        } else if (type == byte.class) {
+          field.setByte(entry, parser.parseByte(columnIds[0]));
+        } else if (type == byte[].class) {
+          field.set(entry, parser.parseByte(columnIds));
+        } else if (type == short.class) {
+          field.setShort(entry, parser.parseShort(columnIds[0]));
+        } else if (type == short[].class) {
+          field.set(entry, parser.parseShort(columnIds));
+        } else if (type == int.class) {
+          field.setInt(entry, parser.parseInt(columnIds[0]));
+        } else if (type == int[].class) {
+          field.set(entry, parser.parseInt(columnIds));
+        } else if (type == long.class) {
+          field.setLong(entry, parser.parseLong(columnIds[0]));
+        } else if (type == long[].class) {
+          field.set(entry, parser.parseLong(columnIds));
+        } else if (type == boolean.class) {
+          field.setBoolean(entry, parser.parseBoolean(columnIds[0]));
+        } else if (type == boolean[].class) {
+          field.set(entry, parser.parseBoolean(columnIds));
+        } else if (type == float.class) {
+          field.setFloat(entry, parser.parseFloat(columnIds[0]));
+        } else if (type == float[].class) {
+          field.set(entry, parser.parseFloat(columnIds));
+        } else if (type == double.class) {
+          field.setDouble(entry, parser.parseDouble(columnIds[0]));
+        } else if (type == double[].class) {
+          field.set(entry, parser.parseDouble(columnIds));
+        } else {
+          throw new ParseException(field, "Cannot parse fields of type %s",
+              org.apache.commons.lang3.ClassUtils.getCanonicalName(type));
+        }
+      } catch (ColumnFormat t) {
+        ParseException parseException = new ParseException(field,
+            "error parsing field %s row: '%s' column: '%s': '%s' as %s",
+            field, key, columnNames[t.columnIndex()], t.columnText(),
+            type.isArray() ? type.getComponentType().getCanonicalName() : type.getCanonicalName());
+        parseException.initCause(t);
+        throw parseException;
+      }
+    }
+  }
+
+  /**
+   * Parses the specified field using it's column definition annotation to
+   * generate a list of column names and indexes associated with them. These
+   * indexes are then stored as a mapping from field to associated column
+   * indexes which can be used to retrieve data from the backing excel.
+   */
+  static void populateColumnIndexes(
+      final Array<Triple<Field, int[], String[]>> columns,
+      final TxtParser parser,
+      final Entry.Column column,
+      final Field field,
+      final String[] TMP
+  ) throws ParseException {
+      final String format = column.format();
+      final String[] values = column.values();
+      final int startIndex = column.startIndex();
+      final int endIndex = column.endIndex();
+      final int columnIndex = column.columnIndex();
+    if (columnIndex >= 0) {
+      final int[] columnIndexes = new int[] { columnIndex };
+      final String[] columnNames = new String[] { null };
+      columns.add(Triple.of(field, columnIndexes, columnNames));
+      log.trace("pushing column <{}>->{}", field, columnIndexes);
+    } else if (format.isEmpty()) {
+      final String fieldName = field.getName();
+      if (values.length > 0) {
+        // values[] used as literal column names
+        checkArrayColumns(field, values.length);
+        String[] columnNames = new String[values.length];
+        for (int i = 0; i < values.length; i++) {
+          columnNames[i] = values[i];
+        }
+
+        putColumns(columns, parser, field, columnNames);
+      } else if (startIndex == 0 && endIndex == 0) {
+        // field name used as literal column name
+        TMP[0] = fieldName;
+        putColumns(columns, parser, field, TMP);
+      } else {
+        // field name + indexes used as column names
+        checkArrayColumns(field, endIndex - startIndex);
+        String[] columnNames = new String[endIndex - startIndex];
+        for (int i = startIndex, j = 0; i < endIndex; i++, j++) {
+          columnNames[j] = fieldName + i;
+        }
+
+        putColumns(columns, parser, field, columnNames);
+      }
+    } else {
+      if (startIndex == 0 && endIndex == 0) {
+        // format used as literal column name
+        TMP[0] = format;
+        putColumns(columns, parser, field, TMP);
+      } else {
+        checkArrayColumns(field, endIndex - startIndex);
+        String[] columnNames = new String[endIndex - startIndex];
+        if (values.length == 0) {
+          // format used in conjunction with indexes as column names
+          // format must contain %d within it, replaced with indexes
+          for (int i = startIndex, j = 0; i < endIndex; i++, j++) {
+            columnNames[j] = String.format(format, i);
+          }
+        } else {
+          // format used in conjunction with values as column names
+          // format must contain as many values as indexes
+          for (int i = 0, s = values.length; i < s; i++) {
+            columnNames[i] = String.format(format, values[i]);
+          }
+        }
+
+        putColumns(columns, parser, field, columnNames);
+      }
+    }
+
+    if (log.debugEnabled()) {
+      StringBuilder builder = new StringBuilder(256);
+      builder.append('{');
+      for (Triple<Field, int[], String[]> pair : columns) {
+        builder
+            .append(pair.getLeft().getName())
+            .append('=')
+            .append(Arrays.toString(pair.getMiddle()))
+            .append(", ");
+      }
+      if (columns.size > 0) builder.setLength(builder.length() - 2);
+      builder.append('}');
+      log.debug("columns: {}", builder.toString());
+    }
+  }
+
+  static void checkArrayColumns(Field field, int length) throws ParseException {
+    if (!field.getType().isArray() && length > 1) {
+      throw new ParseException(field, ""
+          + "field %s corresponds to multiple columns. "
+          + "is it supposed to be an array type?", field);
+    }
+  }
+
+  static int putColumns(
+      Array<Triple<Field, int[], String[]>> columns,
+      TxtParser parser,
+      Field field,
+      String[] columnNames
+  ) {
+    final int index = columns.size;
+    final int[] columnIndexes = parser.columnId(columnNames);
+    columns.add(Triple.of(field, columnIndexes, columnNames));
+    log.trace("pushing columns {}->{}", columnNames, columnIndexes);
+    if (log.warnEnabled()) {
+      for (int i = 0, s = columnIndexes.length; i < s; i++) {
+        if (columnIndexes[i] == -1) {
+          log.warn("Unable to parse column named '{}'", columnNames[i]);
+        }
+      }
+    }
+
+    return index;
   }
 
   static <E extends Entry, S extends Serializer<E>, T extends Excel<E, S>>
@@ -121,10 +418,47 @@ public abstract class Excel<
     throw null;
   }
 
+  static <E extends Entry, T extends Excel<E, ?>>
+  void putIndex(
+      Field primaryKey,
+      Class primaryKeyType,
+      int i,
+      boolean indexed,
+      T excel,
+      E entry
+  ) throws IllegalAccessException {
+    if (indexed) {
+      excel.put(i, entry);
+    } else if (primaryKeyType == int.class) {
+      int id = primaryKey.getInt(entry);
+      excel.put(id, entry);
+    } else if (primaryKeyType == String.class) {
+      String id = (String) primaryKey.get(entry);
+      excel.put(i, entry);
+
+      if (excel.stringToIndex == EMPTY_OBJECT_INT_MAP) excel.stringToIndex = new ObjectIntMap<>();
+      if (!excel.stringToIndex.containsKey(id)) excel.stringToIndex.put(id, i);
+    }
+  }
+
   protected final Class<E> entryClass;
+  protected ObjectIntMap<String> stringToIndex;
+  protected IntMap<E> entries;
+  protected Array<Entry> ordered;
 
   protected Excel(Class<E> entryClass) {
+    this(entryClass, 53);
+  }
+
+  protected Excel(Class<E> entryClass, int initialCapacity) {
+    this(entryClass, initialCapacity, 0.8f);
+  }
+
+  protected Excel(Class<E> entryClass, int initialCapacity, float loadFactor) {
     this.entryClass = entryClass;
+    this.stringToIndex = emptyMap();
+    this.entries = new IntMap<>(initialCapacity, loadFactor);
+    this.ordered = new Array<>(true, (int) (initialCapacity * loadFactor), Entry.class);
   }
 
   public Class<? extends Excel> excelClass() {
@@ -135,12 +469,36 @@ public abstract class Excel<
     return entryClass;
   }
 
+  protected void put(int id, E value) {}
+
+  protected int offset() {
+    return 0;
+  }
+
+  protected void init() {}
+
+  public E get(String id) {
+    return get(index(id));
+  }
+
+  public E get(int id) {
+    return entries.get(id);
+  }
+
+  public int index(String id) {
+    return stringToIndex.get(id, -1);
+  }
+
+  public int size() {
+    return entries.size;
+  }
+
   public abstract E newEntry();
 
   public abstract S newSerializer();
 
   @Override
   public Iterator<E> iterator() {
-    throw new UnsupportedOperationException();
+    return entries.values().iterator();
   }
 }

core/src/main/java/com/riiablo/excel2/ParseException.java

@@ -1,5 +1,9 @@
 package com.riiablo.excel2;
 
+import java.lang.reflect.Field;
+
+import com.badlogic.gdx.utils.Array;
+
 public class ParseException extends Exception {
   ParseException(String message) {
     super(message);
@@ -8,4 +12,59 @@ public class ParseException extends Exception {
   ParseException(String format, Object... args) {
     this(String.format(format, args));
   }
+
+  ParseException(Field field, String format, Object... args) {
+    this(format, args);
+
+    // Formats the leading stack trace element like:
+    //   at com.riiablo.excel.txt.MonStats$Entry.hcIdx2(MonStats.java:0)
+    Class declaringClass = field.getDeclaringClass();
+    StackTraceElement fieldElement = new StackTraceElement(
+        declaringClass.getName(),
+        field.getName(),
+        getRootClass(declaringClass).getSimpleName() + ".java",
+        0); // 0 indicates line 0 -- non-zero required for link parsing in IDEA
+
+    StackTraceElement[] originalStackTrace = getStackTrace();
+    Array<StackTraceElement> stackTrace = new Array<>(
+        true,
+        originalStackTrace.length + 1,
+        StackTraceElement.class);
+    stackTrace.add(fieldElement);
+    stackTrace.addAll(originalStackTrace);
+    setStackTrace(stackTrace.toArray());
+  }
+
+  ParseException(Class clazz, String format, Object... args) {
+    this(format, args);
+
+    // Formats the leading stack trace element like:
+    //   at com.riiablo.excel.txt.MonStats$Entry.hcIdx2(MonStats.java:0)
+    Class declaringClass = clazz.getDeclaringClass();
+    StackTraceElement fieldElement = new StackTraceElement(
+        declaringClass.getName(),
+        clazz.getName(),
+        getRootClass(declaringClass).getSimpleName() + ".java",
+        0); // 0 indicates line 0 -- non-zero required for link parsing in IDEA
+
+    StackTraceElement[] originalStackTrace = getStackTrace();
+    Array<StackTraceElement> stackTrace = new Array<>(
+        true,
+        originalStackTrace.length + 1,
+        StackTraceElement.class);
+    stackTrace.add(fieldElement);
+    stackTrace.addAll(originalStackTrace);
+    setStackTrace(stackTrace.toArray());
+  }
+
+  private static Class getRootClass(Class c) {
+    Class declaringClass = c;
+    for (
+        Class parent = declaringClass;
+        (parent = parent.getDeclaringClass()) != null;) {
+      declaringClass = parent;
+    }
+
+    return declaringClass;
+  }
 }

core/src/main/java/com/riiablo/excel2/txt/MonStats.java

@@ -11,7 +11,7 @@ import com.riiablo.io.ByteOutput;
 @SerializedWith(MonStats.Serializer.class)
 public class MonStats extends Excel<MonStats.Entry, MonStats.Serializer> {
   public MonStats() {
-    super(Entry.class);
+    super(Entry.class, 1543); // 736 entries
   }
 
   @Override

core/src/test/java/com/riiablo/excel2/ExcelTest.java

@@ -10,7 +10,6 @@ import com.riiablo.RiiabloTest;
 import com.riiablo.excel2.txt.MonStats;
 import com.riiablo.logger.Level;
 import com.riiablo.logger.LogManager;
-import com.riiablo.logger.MDC;
 
 public class ExcelTest extends RiiabloTest {
   @org.junit.BeforeClass
@@ -21,11 +20,6 @@ public class ExcelTest extends RiiabloTest {
   @Test
   public void parse_monstats_columns() throws IOException {
     FileHandle handle = Gdx.files.internal("test/monstats.txt");
-    try {
-      MDC.put("excel", handle.path());
-      Excel.loadTxt(new MonStats(), handle);
-    } finally {
-      MDC.remove("excel");
-    }
+    Excel.loadTxt(new MonStats(), handle);
   }
 }