First commit on supporting parquet

pom.xml

@@ -47,12 +47,12 @@
     </organization>
 
     <modules>
-        <module>xtable-api</module>
-        <module>xtable-hudi-support</module>
+       <module>xtable-api</module>
+        <!--     <module>xtable-hudi-support</module>-->
         <module>xtable-core</module>
-        <module>xtable-utilities</module>
-        <module>xtable-aws</module>
-        <module>xtable-hive-metastore</module>
+        <!--    <module>xtable-utilities</module>
+           <module>xtable-aws</module>
+          <module>xtable-hive-metastore</module>-->
     </modules>
 
     <properties>
@@ -304,6 +304,41 @@
                 <version>${delta.hive.version}</version>
             </dependency>
 
+            <!-- Parquet -->
+            <dependency>
+                <groupId>org.apache.parquet</groupId>
+                <artifactId>parquet</artifactId>
+                <version>1.13.0</version>
+                <type>pom</type>
+            </dependency>
+
+            <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-format -->
+            <dependency>
+                <groupId>org.apache.parquet</groupId>
+                <artifactId>parquet-format</artifactId>
+                <version>2.10.0</version>
+            </dependency>
+
+
+            <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
+            <dependency>
+                <groupId>org.apache.parquet</groupId>
+                <artifactId>parquet-hadoop</artifactId>
+                <version>1.13.0</version>
+            </dependency>
+            <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-column -->
+            <dependency>
+                <groupId>org.apache.parquet</groupId>
+                <artifactId>parquet-column</artifactId>
+                <version>1.13.0</version>
+            </dependency>
+            <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-common -->
+            <dependency>
+                <groupId>org.apache.parquet</groupId>
+                <artifactId>parquet-common</artifactId>
+                <version>1.13.0</version>
+            </dependency>
+
             <!-- Spark -->
             <dependency>
                 <groupId>org.apache.spark</groupId>
@@ -794,7 +829,7 @@
                 <configuration>
                     <java>
                         <googleJavaFormat>
-                            <version>${google.java.format.version}</version>
+                            <version>1.19.2</version> <!--   ${google.java.format.version}  -->
                             <style>GOOGLE</style>
                             <groupArtifact>com.google.googlejavaformat:google-java-format</groupArtifact>
                         </googleJavaFormat>

xtable-api/pom.xml

@@ -29,6 +29,12 @@
     <name>XTable Project API</name>
 
     <dependencies>
+        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-column -->
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-column</artifactId>
+            <version>1.13.0</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>

xtable-api/src/main/java/org/apache/xtable/conversion/ConversionConfig.java

@@ -28,6 +28,7 @@
 
 import com.google.common.base.Preconditions;
 
+import org.apache.xtable.model.config.InputPartitionFields;
 import org.apache.xtable.model.sync.SyncMode;
 
 @Value
@@ -42,19 +43,23 @@ public class ConversionConfig {
   Map<TargetTable, List<TargetCatalogConfig>> targetCatalogs;
   // The mode, incremental or snapshot
   SyncMode syncMode;
+  // Input partition config for parquet
+  InputPartitionFields partitions;
 
   @Builder
   ConversionConfig(
       @NonNull SourceTable sourceTable,
       List<TargetTable> targetTables,
       Map<TargetTable, List<TargetCatalogConfig>> targetCatalogs,
-      SyncMode syncMode) {
+      SyncMode syncMode,
+      InputPartitionFields partitions) {
     this.sourceTable = sourceTable;
     this.targetTables = targetTables;
     Preconditions.checkArgument(
         targetTables != null && !targetTables.isEmpty(),
         "Please provide at-least one format to sync");
     this.targetCatalogs = targetCatalogs == null ? Collections.emptyMap() : targetCatalogs;
     this.syncMode = syncMode == null ? SyncMode.INCREMENTAL : syncMode;
+    this.partitions = partitions;
   }
 }

xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java

@@ -34,12 +34,16 @@
 class ExternalTable {
   /** The name of the table. */
   protected final @NonNull String name;
+
   /** The format of the table (e.g. DELTA, ICEBERG, HUDI) */
   protected final @NonNull String formatName;
+
   /** The path to the root of the table or the metadata directory depending on the format */
   protected final @NonNull String basePath;
+
   /** Optional namespace for the table */
   protected final String[] namespace;
+
   /** The configuration for interacting with the catalog that manages this table */
   protected final CatalogConfig catalogConfig;
 

xtable-api/src/main/java/org/apache/xtable/model/catalog/HierarchicalTableIdentifier.java

@@ -23,12 +23,18 @@
  * table names. Some catalogs may omit the catalog name.
  */
 public interface HierarchicalTableIdentifier extends CatalogTableIdentifier {
-  /** @return the catalog name if present, otherwise null */
+  /**
+   * @return the catalog name if present, otherwise null
+   */
   String getCatalogName();
 
-  /** @return the database (or schema) name; required */
+  /**
+   * @return the database (or schema) name; required
+   */
   String getDatabaseName();
 
-  /** @return the table name; required */
+  /**
+   * @return the table name; required
+   */
   String getTableName();
 }

xtable-api/src/main/java/org/apache/xtable/model/catalog/ThreePartHierarchicalTableIdentifier.java

@@ -42,6 +42,7 @@ public class ThreePartHierarchicalTableIdentifier implements HierarchicalTableId
    * name varies depending on the catalogType.
    */
   String catalogName;
+
   /**
    * Catalogs have the ability to group tables logically, databaseName is the identifier for such
    * logical classification. The alternate names for this field include namespace, schemaName etc.

xtable-api/src/main/java/org/apache/xtable/model/config/InputPartitionField.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.model.config;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.Value;
+
+import org.apache.xtable.model.schema.PartitionTransformType;
+
+@Data
+@Value
+@Builder(toBuilder = true)
+public class InputPartitionField {
+  String partitionFieldName;
+  String partitionValue;
+  PartitionTransformType transformType;
+}

xtable-api/src/main/java/org/apache/xtable/model/config/InputPartitionFields.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.model.config;
+
+import java.util.List;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.Value;
+
+@Data
+@Value
+@Builder(toBuilder = true)
+public class InputPartitionFields {
+  String sourceField;
+  List<InputPartitionField> partitions;
+}

xtable-api/src/main/java/org/apache/xtable/model/schema/InternalField.java

@@ -43,9 +43,11 @@ public class InternalField {
   // The id field for the field. This is used to identify the field in the schema even after
   // renames.
   Integer fieldId;
+
   // represents the fully qualified path to the field (dot separated)
   @Getter(lazy = true)
   String path = createPath();
+
   // splits the dot separated path into parts
   @Getter(lazy = true)
   String[] pathParts = splitPath();

xtable-api/src/main/java/org/apache/xtable/model/schema/InternalSchema.java

@@ -75,7 +75,8 @@ public enum MetadataKey {
 
   public enum MetadataValue {
     MICROS,
-    MILLIS
+    MILLIS,
+    NANOS
   }
 
   public static final String XTABLE_LOGICAL_TYPE = "xtableLogicalType";

xtable-api/src/main/java/org/apache/xtable/model/stat/ColumnStat.java

@@ -17,11 +17,13 @@
  */
 
 package org.apache.xtable.model.stat;
-
+import java.util.Set;
 import lombok.Builder;
 import lombok.Value;
 
 import org.apache.xtable.model.schema.InternalField;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.statistics.Statistics;
 
 /**
  * Captures column level statistics for a field.
@@ -36,4 +38,7 @@ public class ColumnStat {
   long numNulls;
   long numValues;
   long totalSize;
+  long uncompressedSize;
+  Set<Encoding> encodings;
+  Statistics statistics;
 }

xtable-api/src/main/java/org/apache/xtable/model/storage/InternalDataFile.java

@@ -49,4 +49,8 @@ public class InternalDataFile {
   @Builder.Default @NonNull List<ColumnStat> columnStats = Collections.emptyList();
   // last modified time in millis since epoch
   long lastModified;
+
+  public static InternalDataFileBuilder builderFrom(InternalDataFile dataFile) {
+    return dataFile.toBuilder();
+  }
 }

xtable-api/src/main/java/org/apache/xtable/model/storage/TableFormat.java

@@ -27,8 +27,9 @@ public class TableFormat {
   public static final String HUDI = "HUDI";
   public static final String ICEBERG = "ICEBERG";
   public static final String DELTA = "DELTA";
+  public static final String PARQUET = "PARQUET";
 
   public static String[] values() {
-    return new String[] {"HUDI", "ICEBERG", "DELTA"};
+    return new String[] {"HUDI", "ICEBERG", "DELTA", "PARQUET"};
   }
 }

xtable-core/pom.xml

@@ -57,6 +57,12 @@
         </dependency>
 
         <!-- Avro -->
+        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro -->
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-avro</artifactId>
+            <version>1.15.0</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>