Second iteration of revision of excel-refactor

core/src/main/java/com/riiablo/excel2/Excel.java

@@ -0,0 +1,157 @@
+package com.riiablo.excel2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.Iterator;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import com.badlogic.gdx.files.FileHandle;
+
+import com.riiablo.logger.LogManager;
+import com.riiablo.logger.Logger;
+
+/**
+ * Root class of an excel table.
+ */
+public abstract class Excel<
+    E extends Excel.Entry,
+    S extends Serializer<E>
+>
+    implements Iterable<E>
+{
+  private static final Logger log = LogManager.getLogger(Excel.class);
+
+  /**
+   * Tags the specified excel as indexed. Indexed excels apply a 1-to-1
+   * assignment of row index to entry index. Used in the case where the excel
+   * does not include a primary key column.
+   */
+  @Target(ElementType.TYPE)
+  @Retention(RetentionPolicy.RUNTIME)
+  public @interface Indexed {}
+
+  /**
+   * Root class of an excel entry.
+   */
+  public static abstract class Entry {
+    /**
+     * Tags the specified field as a column within the excel table.
+     */
+    @Target(ElementType.FIELD)
+    @Retention(RetentionPolicy.RUNTIME)
+    public @interface Column {
+      /**
+       * Start index of {@link #format()} (inclusive)
+       */
+      int startIndex() default 0;
+
+      /**
+       * End index of {@link #format()} (exclusive)
+       */
+      int endIndex() default 0;
+
+      /**
+       * String format of column name, {@code ""} to use field name
+       * <p>
+       * <p>Examples:
+       * <ul>
+       * <li>{@code "class"}
+       * <li>{@code "Transform Color"}
+       * <li>{@code "Level%s"}
+       * <li>{@code "Skill %d"}
+       */
+      String format() default "";
+
+      /**
+       * Index values of format in the case of non-numerical indexes.
+       * <p>
+       * <p>Examples:
+       * <ul>
+       * <li>{@code {"", "(N)", "(H)"}}
+       * <li>{@code {"r", "g", "b"}}
+       * <li>{@code {"Min", "Max", "MagicMin", "MagicMax", "MagicLvl"}}
+       */
+      String[] values() default {};
+
+      /**
+       * Manually sets the column index. Used in the case where a column might
+       * not be named.
+       */
+      @Deprecated
+      int columnIndex() default -1;
+
+      /**
+       * Whether or not to store value within the generated bin.
+       */
+      boolean bin() default true;
+
+      /**
+       * Tags the column as a foreign key in the specified excel.
+       */
+      Class<? extends Excel> foreignKey() default Excel.class;
+
+      /**
+       * Tags the column as a primary key for this excel.
+       */
+      boolean primaryKey() default false;
+    }
+  }
+
+  public static <E extends Entry, S extends Serializer<E>, T extends Excel<E, S>>
+  T load(T excel, FileHandle txt) throws IOException {
+    return load(excel, txt, null);
+  }
+
+  public static <E extends Entry, S extends Serializer<E>, T extends Excel<E, S>>
+  T load(T excel, FileHandle txt, FileHandle bin) throws IOException {
+    throw null;
+  }
+
+  static <E extends Entry, S extends Serializer<E>, T extends Excel<E, S>>
+  T loadTxt(T excel, FileHandle handle) throws IOException {
+    InputStream in = handle.read();
+    try {
+      TxtParser parser = TxtParser.parse(in);
+      return loadTxt(excel, parser);
+    } catch (IllegalAccessException t) {
+      log.error("Unable to load {} as {}: {}",
+          handle,
+          excel.getClass().getCanonicalName(),
+          ExceptionUtils.getRootCauseMessage(t),
+          t);
+      return ExceptionUtils.wrapAndThrow(t);
+    } finally {
+      IOUtils.closeQuietly(in);
+    }
+  }
+
+  static <E extends Entry, S extends Serializer<E>, T extends Excel<E, S>>
+  T loadTxt(T excel, TxtParser parser) throws IOException, IllegalAccessException {
+    throw null;
+  }
+
+  static <E extends Entry, S extends Serializer<E>, T extends Excel<E, S>>
+  T loadBin(T excel, FileHandle handle) {
+    throw null;
+  }
+
+  protected final Class<E> entryClass;
+
+  protected Excel(Class<E> entryClass) {
+    this.entryClass = entryClass;
+  }
+
+  public abstract E newEntry();
+
+  public abstract S newSerializer();
+
+  @Override
+  public Iterator<E> iterator() {
+    throw new UnsupportedOperationException();
+  }
+}

core/src/main/java/com/riiablo/excel2/Serializer.java

@@ -0,0 +1,9 @@
+package com.riiablo.excel2;
+
+import com.riiablo.io.ByteInput;
+import com.riiablo.io.ByteOutput;
+
+public interface Serializer<T extends Excel.Entry> {
+  void readBin(T entry, ByteInput in);
+  void writeBin(T entry, ByteOutput out);
+}

core/src/main/java/com/riiablo/excel2/TxtParser.java

@@ -0,0 +1,300 @@
+package com.riiablo.excel2;
+
+import io.netty.util.AsciiString;
+import io.netty.util.CharsetUtil;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.commons.io.IOUtils;
+
+import com.badlogic.gdx.utils.Array;
+import com.badlogic.gdx.utils.ByteArray;
+import com.badlogic.gdx.utils.IntArray;
+import com.badlogic.gdx.utils.ObjectIntMap;
+
+import com.riiablo.logger.LogManager;
+import com.riiablo.logger.Logger;
+
+public class TxtParser {
+  private static final Logger log = LogManager.getLogger(TxtParser.class);
+
+  private static final int HT = '\t';
+  private static final int CR = '\r';
+  private static final int LF = '\n';
+
+  private static final byte[] TO_UPPER;
+  static {
+    TO_UPPER = new byte[1 << Byte.SIZE];
+    for (int i = 0; i < TO_UPPER.length; i++) {
+      TO_UPPER[i] = (byte) i;
+    }
+
+    for (int i = 'a'; i <= 'z'; i++) {
+      TO_UPPER[i] &= ~0x20;
+    }
+  }
+
+  private static final AsciiString EXPANSION = AsciiString.cached("EXPANSION");
+
+  public static TxtParser parse(InputStream in) throws IOException {
+    return parse(in, 8192);
+  }
+
+  public static TxtParser parse(InputStream in, int bufferSize) throws IOException {
+    return new TxtParser(in, bufferSize);
+  }
+
+  final int numColumns;
+  final Array<String> columnNames;
+  final ObjectIntMap<String> columnIds;
+
+  BufferedInputStream in;
+  AsciiString line;
+  int index;
+  final ByteArray cache;
+  final IntArray tokenOffsets;
+  int[] tokenOffsetsCache;
+  int numTokens;
+
+  TxtParser(InputStream in, int bufferSize) throws IOException {
+    this.in = IOUtils.buffer(in, bufferSize);
+
+    cache = new ByteArray(512);
+    columnNames = new Array<>();
+    columnIds = new ObjectIntMap<>();
+    numColumns = parseColumnNames();
+
+    log.info("numColumns: {}", numColumns);
+    log.debug("columnNames: {}", columnNames);
+    log.trace("columnIds: {}", columnIds);
+
+    tokenOffsets = new IntArray();
+  }
+
+  private static String toString(ByteArray array) {
+    String stringValue = new String(array.items, 0, array.size, CharsetUtil.US_ASCII);
+    array.clear();
+    return stringValue;
+  }
+
+  private int parseColumnNames() throws IOException {
+    for (int i; (i = in.read()) != -1;) {
+      switch (i) {
+        case HT:
+          putColumnName(toString(cache));
+          break;
+        case CR:
+          in.skip(1);
+        case LF:
+          putColumnName(toString(cache));
+          return columnNames.size;
+        default:
+          cache.add((byte) Character.toUpperCase(i));
+      }
+    }
+
+    throw new IOException("Unexpected end of file while parsing column names");
+  }
+
+  private void putColumnName(String columnName) {
+    if (!columnIds.containsKey(columnName)) {
+      columnIds.put(columnName, columnNames.size);
+    }
+
+    columnNames.add(columnName);
+  }
+
+  public int cacheLine() throws IOException {
+    cache.clear();
+    tokenOffsets.clear();
+    tokenOffsets.add(0);
+    index++;
+copy:
+    for (int i; (i = in.read()) != -1;) {
+      switch (i) {
+        case HT:
+          tokenOffsets.add(cache.size);
+          break;
+        case CR:
+          in.skip(1);
+        case LF:
+          tokenOffsets.add(cache.size);
+          break copy;
+        default:
+          cache.add(TO_UPPER[i]);
+      }
+    }
+
+    numTokens = tokenOffsets.size - 1;
+    tokenOffsetsCache = tokenOffsets.items;
+    line = new AsciiString(cache.items, 0, cache.size, false);
+    log.debug("line: {}", line);
+    if (line.contentEqualsIgnoreCase(EXPANSION)) {
+      log.trace("skipping row {}: {} is an ignored symbol", index, EXPANSION);
+      return cacheLine();
+    }
+
+    if (numTokens != numColumns) {
+      log.warn("skipping row {}: contains {} tokens, expected {}; tokens: {}",
+          index, numTokens, numColumns, getTokens());
+      return cacheLine();
+    }
+
+    if (log.traceEnabled()) {
+      final int[] tokenOffsets = this.tokenOffsets.items;
+      for (int i = 1, j = tokenOffsets[i - 1], s = this.tokenOffsets.size; i < s; i++) {
+        final int tokenOffset = tokenOffsets[i];
+        log.trace("{}={}", getColumnName(i - 1), line.subSequence(j, tokenOffset, false));
+        j = tokenOffset;
+      }
+    }
+
+    return tokenOffsets.size - 1;
+  }
+
+  public int getNumColumns() {
+    return numColumns;
+  }
+
+  public String[] getColumnNames() {
+    final String[] columnNames = new String[numColumns];
+    for (int i = 0; i < numColumns; i++) columnNames[i] = getColumnName(i);
+    return columnNames;
+  }
+
+  public String getColumnName(int i) {
+    return columnNames.get(i).toString();
+  }
+
+  public String getRowName() {
+    return parseString(0);
+  }
+
+  public int getColumnId(String columnName) {
+    return columnIds.get(columnName.toUpperCase(), -1);
+  }
+
+  public int[] getColumnId(String[] columnNames) {
+    final int numColumns = columnNames.length;
+    final int[] columnIds = new int[numColumns];
+    for (int i = 0; i < numColumns; i++) columnIds[i] = getColumnId(columnNames[i]);
+    return columnIds;
+  }
+
+  public int getNumTokens() {
+    return numTokens;
+  }
+
+  public AsciiString[] getTokens() {
+    final int numTokens = getNumTokens();
+    final AsciiString[] tokens = new AsciiString[numTokens];
+    for (int i = 0; i < numTokens; i++) tokens[i] = getToken(i);
+    return tokens;
+  }
+
+  public AsciiString getToken(int i) {
+    final int[] tokenOffsets = tokenOffsetsCache;
+    return line.subSequence(tokenOffsets[i], tokenOffsets[i + 1]);
+  }
+
+  public byte parseByte(int i) {
+    final int[] tokenOffsets = tokenOffsetsCache;
+    final int intValue = line.parseInt(tokenOffsets[i], tokenOffsets[i + 1]);
+    final byte result = (byte) intValue;
+    if (result != intValue) {
+      throw new NumberFormatException(line.subSequence(tokenOffsets[i], tokenOffsets[i + 1], false).toString());
+    }
+
+    return result;
+  }
+
+  public short parseShort(int i) {
+    final int[] tokenOffsets = tokenOffsetsCache;
+    return line.parseShort(tokenOffsets[i], tokenOffsets[i + 1]);
+  }
+
+  public int parseInt(int i) {
+    final int[] tokenOffsets = tokenOffsetsCache;
+    return line.parseInt(tokenOffsets[i], tokenOffsets[i + 1]);
+  }
+
+  public long parseLong(int i) {
+    final int[] tokenOffsets = tokenOffsetsCache;
+    return line.parseLong(tokenOffsets[i], tokenOffsets[i + 1]);
+  }
+
+  public boolean parseBoolean(int i) {
+    final int[] tokenOffsets = tokenOffsetsCache;
+    final int intValue = line.parseInt(tokenOffsets[i], tokenOffsets[i + 1]);
+    if ((intValue & 1) != intValue) {
+      log.warn("boolean exceeds boolean radix at {}:{} (\"{}\", \"{}\"): {}",
+          index, i, getRowName(), getColumnName(i), intValue);
+    }
+
+    return intValue != 0;
+  }
+
+  public float parseFloat(int i) {
+    final int[] tokenOffsets = tokenOffsetsCache;
+    return line.parseFloat(tokenOffsets[i], tokenOffsets[i + 1]);
+  }
+
+  public double parseDouble(int i) {
+    final int[] tokenOffsets = tokenOffsetsCache;
+    return line.parseDouble(tokenOffsets[i], tokenOffsets[i + 1]);
+  }
+
+  public String parseString(int i) {
+    final int[] tokenOffsets = tokenOffsetsCache;
+    return line.toString(tokenOffsets[i], tokenOffsets[i + 1]);
+  }
+
+  public byte[] parseByte(int[] columns) {
+    final int length = columns.length;
+    byte[] values = new byte[length];
+    for (int i = 0; i < length; i++) values[i] = parseByte(columns[i]);
+    return values;
+  }
+
+  public short[] parseShort(int[] columns) {
+    final int length = columns.length;
+    short[] values = new short[length];
+    for (int i = 0; i < length; i++) values[i] = parseShort(columns[i]);
+    return values;
+  }
+
+  public long[] parseLong(int[] columns) {
+    final int length = columns.length;
+    long[] values = new long[length];
+    for (int i = 0; i < length; i++) values[i] = parseLong(columns[i]);
+    return values;
+  }
+
+  public boolean[] parseBoolean(int[] columns) {
+    final int length = columns.length;
+    boolean[] values = new boolean[length];
+    for (int i = 0; i < length; i++) values[i] = parseBoolean(columns[i]);
+    return values;
+  }
+
+  public float[] parseFloat(int[] columns) {
+    final int length = columns.length;
+    float[] values = new float[length];
+    for (int i = 0; i < length; i++) values[i] = parseFloat(columns[i]);
+    return values;
+  }
+
+  public double[] parseDouble(int[] columns) {
+    final int length = columns.length;
+    double[] values = new double[length];
+    for (int i = 0; i < length; i++) values[i] = parseDouble(columns[i]);
+    return values;
+  }
+
+  public String[] parseString(int[] columns) {
+    final int length = columns.length;
+    String[] values = new String[length];
+    for (int i = 0; i < length; i++) values[i] = parseString(columns[i]);
+    return values;
+  }
+}

core/src/main/java/com/riiablo/excel2/txt/MonStats.java

@@ -0,0 +1,31 @@
+package com.riiablo.excel2.txt;
+
+import com.riiablo.excel2.Excel;
+import com.riiablo.io.ByteInput;
+import com.riiablo.io.ByteOutput;
+
+public class MonStats extends Excel<MonStats.Entry, MonStats.Serializer> {
+  public MonStats() {
+    super(Entry.class);
+  }
+
+  @Override
+  public Entry newEntry() {
+    return new Entry();
+  }
+
+  @Override
+  public Serializer newSerializer() {
+    return new Serializer();
+  }
+
+  public static class Entry extends Excel.Entry {
+    @Column public String Id;
+    @Column public int hcIdx;
+  }
+
+  public static class Serializer implements com.riiablo.excel2.Serializer<Entry> {
+    @Override public void readBin(Entry entry, ByteInput in) {}
+    @Override public void writeBin(Entry entry, ByteOutput out) {}
+  }
+}

core/src/test/java/com/riiablo/excel2/ExcelTest.java

@@ -0,0 +1,24 @@
+package com.riiablo.excel2;
+
+import java.io.IOException;
+import org.junit.Test;
+
+import com.badlogic.gdx.Gdx;
+import com.badlogic.gdx.files.FileHandle;
+
+import com.riiablo.RiiabloTest;
+import com.riiablo.logger.Level;
+import com.riiablo.logger.LogManager;
+
+public class ExcelTest extends RiiabloTest {
+  @org.junit.BeforeClass
+  public static void before() {
+    LogManager.setLevel("com.riiablo.excel2", Level.TRACE);
+  }
+
+  @Test
+  public void parse_monstats_columns() throws IOException {
+    FileHandle handle = Gdx.files.internal("test/monstats.txt");
+    Excel.loadTxt(null, handle);
+  }
+}

core/src/test/java/com/riiablo/excel2/TxtParserTest.java

@@ -0,0 +1,52 @@
+package com.riiablo.excel2;
+
+import java.io.IOException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.badlogic.gdx.Gdx;
+import com.badlogic.gdx.files.FileHandle;
+
+import com.riiablo.RiiabloTest;
+import com.riiablo.logger.Level;
+import com.riiablo.logger.LogManager;
+
+public class TxtParserTest extends RiiabloTest {
+  @org.junit.BeforeClass
+  public static void before() {
+    LogManager.setLevel("com.riiablo.excel2", Level.TRACE);
+  }
+
+  @Test
+  public void parse_monstats_columns() throws IOException {
+    FileHandle handle = Gdx.files.internal("test/monstats.txt");
+    TxtParser.parse(handle.read());
+  }
+
+  @Test
+  public void parse_monstats_first() throws IOException {
+    FileHandle handle = Gdx.files.internal("test/monstats.txt");
+    TxtParser parser = TxtParser.parse(handle.read());
+    parser.cacheLine();
+  }
+
+  @Test
+  public void parse_monstats_first_2() throws IOException {
+    LogManager.setLevel("com.riiablo.excel2", Level.DEBUG);
+    FileHandle handle = Gdx.files.internal("test/monstats.txt");
+    TxtParser parser = TxtParser.parse(handle.read());
+    parser.cacheLine();
+    parser.cacheLine();
+    LogManager.setLevel("com.riiablo.excel2", Level.TRACE);
+  }
+
+  @Test
+  public void parse_monstats_parseInt() throws IOException {
+    LogManager.setLevel("com.riiablo.excel2", Level.DEBUG);
+    FileHandle handle = Gdx.files.internal("test/monstats.txt");
+    TxtParser parser = TxtParser.parse(handle.read());
+    parser.cacheLine();
+    Assert.assertEquals(0, parser.parseInt(1));
+    LogManager.setLevel("com.riiablo.excel2", Level.TRACE);
+  }
+}