DRILL-8542: Support Paimon Format Plugin

contrib/format-paimon/README.md

@@ -0,0 +1,96 @@
+# Apache Paimon format plugin
+
+This format plugin enables Drill to query Apache Paimon tables.
+
+Unlike regular format plugins, the Paimon table is a folder with data and metadata files, but Drill checks the presence
+of the `snapshot` directory and `schema` directory to ensure that the table is a Paimon one.
+
+Drill supports reading all formats of Paimon tables currently supported via Paimon Java API: Parquet and ORC.
+No need to provide actual table format, it will be discovered automatically.
+
+For details related to Apache Paimon table format, please refer to [official docs](https://paimon.apache.org/).
+
+## Supported optimizations and features
+
+### Project pushdown
+
+This format plugin supports project pushdown optimization.
+
+For the case of project pushdown, only columns specified in the query will be read. In conjunction with
+column-oriented formats like Parquet or ORC, it allows improving reading performance significantly.
+
+### Filter pushdown
+
+This format plugin supports filter pushdown optimization.
+
+For the case of filter pushdown, expressions supported by Paimon API will be pushed down, so only data that matches
+the filter expression will be read.
+
+### Limit pushdown
+
+This format plugin supports limit pushdown optimization.
+
+The limit is pushed down to Paimon scan planning to reduce the amount of data read.
+
+### Querying table metadata
+
+Apache Drill provides the ability to query table metadata exposed by Paimon.
+
+At this point, Apache Paimon has the following metadata kinds:
+
+* SNAPSHOTS
+* SCHEMAS
+* FILES
+* MANIFESTS
+
+To query specific metadata, just add the `#metadata_name` suffix to the table location, like in the following example:
+
+```sql
+SELECT *
+FROM dfs.tmp.`testTable#snapshots`;
+```
+
+### Querying specific table versions (time travel)
+
+Apache Paimon has the ability to track the table modifications and read specific version before or after modifications
+or modifications itself.
+
+This format plugin embraces this ability and provides an easy-to-use way of triggering it.
+
+The following ways of specifying table version are supported:
+
+- `snapshotId` - id of the specific snapshot
+- `snapshotAsOfTime` - the most recent snapshot as of the given time in milliseconds
+
+Table function can be used to specify one of the above configs in the following way:
+
+```sql
+SELECT *
+FROM table(dfs.tmp.testTable(type => 'paimon', snapshotId => 1));
+
+SELECT *
+FROM table(dfs.tmp.testTable(type => 'paimon', snapshotAsOfTime => 1736345510000));
+```
+
+Note: `snapshotId` and `snapshotAsOfTime` are mutually exclusive and cannot be specified at the same time.
+
+## Configuration
+
+The only required configuration option is:
+
+- `type` - format plugin type, should be `'paimon'`
+
+Note: `snapshotId` and `snapshotAsOfTime` for time travel queries are specified at query time using the `table()` function.
+
+### Format config example:
+
+```json
+{
+  "type": "file",
+  "formats": {
+    "paimon": {
+      "type": "paimon"
+    }
+  }
+}
+```

contrib/format-paimon/pom.xml

@@ -0,0 +1,123 @@
+<?xml version="1.0"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>drill-contrib-parent</artifactId>
+    <groupId>org.apache.drill.contrib</groupId>
+    <version>1.23.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>drill-paimon-format</artifactId>
+
+  <name>Drill : Contrib : Format : Paimon</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.drill.exec</groupId>
+      <artifactId>drill-java-exec</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.paimon</groupId>
+      <artifactId>paimon-core</artifactId>
+      <version>${paimon.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.paimon</groupId>
+      <artifactId>paimon-common</artifactId>
+      <version>${paimon.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.paimon</groupId>
+      <artifactId>paimon-api</artifactId>
+      <version>${paimon.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.paimon</groupId>
+      <artifactId>paimon-codegen-loader</artifactId>
+      <version>${paimon.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.paimon</groupId>
+      <artifactId>paimon-format</artifactId>
+      <version>${paimon.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.paimon</groupId>
+      <artifactId>paimon-shade-jackson-2</artifactId>
+      <version>2.14.2-0.8.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.paimon</groupId>
+      <artifactId>paimon-shade-guava-30</artifactId>
+      <version>30.1.1-jre-0.8.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.paimon</groupId>
+      <artifactId>paimon-shade-caffeine-2</artifactId>
+      <version>2.9.3-0.8.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.paimon</groupId>
+      <artifactId>paimon-shade-netty-4</artifactId>
+      <version>4.1.100.Final-0.8.0</version>
+    </dependency>
+    <dependency>
+      <groupId>io.airlift</groupId>
+      <artifactId>aircompressor</artifactId>
+      <version>0.27</version>
+    </dependency>
+    <dependency>
+      <groupId>org.lz4</groupId>
+      <artifactId>lz4-java</artifactId>
+      <version>1.8.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.github.luben</groupId>
+      <artifactId>zstd-jni</artifactId>
+      <version>1.5.5-11</version>
+    </dependency>
+    <dependency>
+      <groupId>org.xerial.snappy</groupId>
+      <artifactId>snappy-java</artifactId>
+      <version>1.1.8.4</version>
+    </dependency>
+
+    <!-- Test dependency -->
+    <dependency>
+      <groupId>org.apache.drill.exec</groupId>
+      <artifactId>drill-java-exec</artifactId>
+      <classifier>tests</classifier>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.drill</groupId>
+      <artifactId>drill-common</artifactId>
+      <classifier>tests</classifier>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+</project>

contrib/format-paimon/src/main/java/org/apache/drill/exec/store/paimon/PaimonCompleteWork.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.paimon;
+
+import org.apache.drill.exec.store.schedule.CompleteWork;
+import org.apache.drill.exec.store.schedule.EndpointByteMap;
+import org.apache.paimon.table.source.Split;
+
+public class PaimonCompleteWork implements CompleteWork {
+  private final EndpointByteMap byteMap;
+
+  private final Split split;
+
+  private final long totalBytes;
+
+  public PaimonCompleteWork(EndpointByteMap byteMap, Split split) {
+    this.byteMap = byteMap;
+    this.split = split;
+    long rowCount = split.rowCount();
+    this.totalBytes = rowCount > 0 ? rowCount : 1;
+  }
+
+  public Split getSplit() {
+    return split;
+  }
+
+  public long getRowCount() {
+    return split.rowCount();
+  }
+
+  @Override
+  public long getTotalBytes() {
+    return totalBytes;
+  }
+
+  @Override
+  public EndpointByteMap getByteMap() {
+    return byteMap;
+  }
+
+  @Override
+  public int compareTo(CompleteWork o) {
+    return 0;
+  }
+}