diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java index 6cd93adf75..4c1245a3bf 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java @@ -25,6 +25,7 @@ import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.row.InternalRow; import org.apache.fluss.row.decode.FixedSchemaDecoder; +import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.utils.CopyOnWriteMap; import org.apache.fluss.utils.concurrent.FutureUtils; @@ -37,6 +38,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; import static org.apache.fluss.utils.Preconditions.checkArgument; /** Abstract lookuper implementation for common methods. */ @@ -51,6 +53,8 @@ abstract class AbstractLookuper implements Lookuper { private final SchemaGetter schemaGetter; + private final KvValueLayout kvValueLayout; + /** * Cache for row decoders for different schema ids. Use CopyOnWriteMap for fast access, as it is * not frequently updated. @@ -67,12 +71,20 @@ abstract class AbstractLookuper implements Lookuper { this.lookupClient = lookupClient; this.targetSchemaId = (short) tableInfo.getSchemaId(); this.schemaGetter = schemaGetter; + this.kvValueLayout = + KvValueLayout.forKvFormatVersion( + tableInfo + .getTableConfig() + .getKvFormatVersion() + .orElse(KV_FORMAT_VERSION_2)); this.decoders = new CopyOnWriteMap<>(); // initialize the decoder for the same schema this.decoders.put( targetSchemaId, new FixedSchemaDecoder( - tableInfo.getTableConfig().getKvFormat(), tableInfo.getSchema())); + tableInfo.getTableConfig().getKvFormat(), + tableInfo.getSchema(), + kvValueLayout)); } protected void handleLookupResponse( @@ -85,7 +97,7 @@ protected void handleLookupResponse( continue; } MemorySegment memorySegment = MemorySegment.wrap(valueBytes); - short schemaId = memorySegment.getShort(0); + short schemaId = kvValueLayout.readSchemaId(memorySegment); if (targetSchemaId != schemaId) { allTargetSchema = false; if (!decoders.containsKey(schemaId)) { @@ -141,7 +153,7 @@ protected LookupResult processAllTargetSchemaRows(List valueList) protected LookupResult processSchemaMismatchedRows(List valueList) { List rowList = new ArrayList<>(valueList.size()); for (MemorySegment value : valueList) { - short schemaId = value.getShort(0); + short schemaId = kvValueLayout.readSchemaId(value); FixedSchemaDecoder decoder = decoders.get(schemaId); checkArgument(decoder != null, "Decoder for schema id %s not found", schemaId); InternalRow row = decoder.decode(value); @@ -161,7 +173,7 @@ protected LookupResult processSchemaRequestedRows( // process the value list to convert to target schema List rowList = new ArrayList<>(valueList.size()); for (MemorySegment value : valueList) { - short schemaId = value.getShort(0); + short schemaId = kvValueLayout.readSchemaId(value); FixedSchemaDecoder decoder = decoders.computeIfAbsent( schemaId, @@ -170,7 +182,8 @@ protected LookupResult processSchemaRequestedRows( return new FixedSchemaDecoder( tableInfo.getTableConfig().getKvFormat(), sourceSchema, - tableInfo.getSchema()); + tableInfo.getSchema(), + kvValueLayout); }); InternalRow row = decoder.decode(value); rowList.add(row); diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java index 51f12abc7c..e7ed39751f 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java @@ -228,6 +228,10 @@ public BatchScanner createBatchScanner(TableBucket tableBucket, long snapshotId) snapshotMeta.getSnapshotFiles(), projectedColumns, scannerTmpDir, + tableInfo + .getTableConfig() + .getKvFormatVersion() + .orElse(ConfigOptions.KV_FORMAT_VERSION_2), tableInfo.getTableConfig().getKvFormat(), conn.getOrCreateRemoteFileDownloader()); } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvBatchScanner.java index 2062f9c1fc..7250cd0ee2 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvBatchScanner.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvBatchScanner.java @@ -30,6 +30,7 @@ import org.apache.fluss.record.ValueRecordReadContext; import org.apache.fluss.row.InternalRow; import org.apache.fluss.row.ProjectedRow; +import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.PbScanReqForBucket; import org.apache.fluss.rpc.messages.ScanKvRequest; @@ -56,6 +57,8 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; + /** * A {@link BatchScanner} that streams every live row of a single primary-key bucket from the tablet * server's RocksDB state via a sequence of {@code ScanKv} RPCs. The scan has snapshot isolation: @@ -109,7 +112,13 @@ public KvBatchScanner( this.projectedColumns = projectedColumns; this.readContext = ValueRecordReadContext.createReadContext( - schemaGetter, tableInfo.getTableConfig().getKvFormat()); + schemaGetter, + tableInfo.getTableConfig().getKvFormat(), + KvValueLayout.forKvFormatVersion( + tableInfo + .getTableConfig() + .getKvFormatVersion() + .orElse(KV_FORMAT_VERSION_2))); } @Nullable diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java index 52a2c36fb3..e2d4651340 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java @@ -83,6 +83,7 @@ public class KvSnapshotBatchScanner implements BatchScanner { private final Path snapshotLocalDirectory; private final RemoteFileDownloader remoteFileDownloader; + private final int kvFormatVersion; private final KvFormat kvFormat; private final ReentrantLock lock = new ReentrantLock(); @@ -104,6 +105,7 @@ public KvSnapshotBatchScanner( List fsPathAndFileNames, @Nullable int[] projectedFields, String scannerTmpDir, + int kvFormatVersion, KvFormat kvFormat, RemoteFileDownloader remoteFileDownloader) { this.targetSchema = targetSchema; @@ -112,6 +114,7 @@ public KvSnapshotBatchScanner( this.tableBucket = tableBucket; this.fsPathAndFileNames = fsPathAndFileNames; this.projectedFields = projectedFields; + this.kvFormatVersion = kvFormatVersion; this.kvFormat = kvFormat; // create a directory to store the snapshot files this.snapshotLocalDirectory = @@ -220,7 +223,8 @@ private void initReaderAsynchronously() { projectedFields, targetSchemaId, targetSchema, - schemaGetter); + schemaGetter, + kvFormatVersion); readerIsReady.signalAll(); } catch (Throwable e) { IOUtils.closeQuietly(closeableRegistry); diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java index 26e097fefc..01cfecfbce 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java @@ -34,6 +34,7 @@ import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.InternalRow; import org.apache.fluss.row.ProjectedRow; +import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.LimitScanRequest; import org.apache.fluss.rpc.messages.LimitScanResponse; @@ -57,6 +58,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; + /** A {@link BatchScanner} implementation that scans a limited number of records from a table. */ public class LimitBatchScanner implements BatchScanner { @@ -67,6 +70,7 @@ public class LimitBatchScanner implements BatchScanner { private final CompletableFuture scanFuture; private final SchemaGetter schemaGetter; private final KvFormat kvFormat; + private final KvValueLayout kvValueLayout; private final int targetSchemaId; /** The chunked allocation manager factory to reuse memory for arrow log write batch. */ private final ChunkedAllocationManager.ChunkedFactory chunkedFactory; @@ -119,6 +123,12 @@ public LimitBatchScanner( this.scanFuture = gateway.limitScan(limitScanRequest); this.kvFormat = tableInfo.getTableConfig().getKvFormat(); + this.kvValueLayout = + KvValueLayout.forKvFormatVersion( + tableInfo + .getTableConfig() + .getKvFormatVersion() + .orElse(KV_FORMAT_VERSION_2)); this.endOfInput = false; this.chunkedFactory = new ChunkedAllocationManager.ChunkedFactory(); } @@ -163,7 +173,7 @@ private List parseLimitScanResponse(LimitScanResponse limitScanResp DefaultValueRecordBatch valueRecords = DefaultValueRecordBatch.pointToByteBuffer(recordsBuffer); ValueRecordReadContext readContext = - ValueRecordReadContext.createReadContext(schemaGetter, kvFormat); + ValueRecordReadContext.createReadContext(schemaGetter, kvFormat, kvValueLayout); for (ValueRecord record : valueRecords.records(readContext)) { InternalRow row = record.getRow(); if (targetSchemaId != record.schemaId()) { diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java index 8a52814f30..df9d5422d4 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java @@ -27,6 +27,7 @@ import org.apache.fluss.rocksdb.RocksIteratorWrapper; import org.apache.fluss.row.InternalRow; import org.apache.fluss.row.ProjectedRow; +import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.row.encode.ValueDecoder; import org.apache.fluss.utils.CloseableIterator; import org.apache.fluss.utils.CloseableRegistry; @@ -80,12 +81,15 @@ class SnapshotFilesReader implements CloseableIterator { @Nullable int[] projectedFields, int targetSchemaId, Schema targetSchema, - SchemaGetter schemaGetter) + SchemaGetter schemaGetter, + int kvFormatVersion) throws IOException { this.targetSchemaId = targetSchemaId; this.targetSchema = targetSchema; this.schemaGetter = schemaGetter; - this.valueDecoder = new ValueDecoder(schemaGetter, kvFormat); + this.valueDecoder = + new ValueDecoder( + schemaGetter, kvFormat, KvValueLayout.forKvFormatVersion(kvFormatVersion)); this.projectedFields = projectedFields; closeableRegistry = new CloseableRegistry(); try { diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 2f6617b20f..4c203394b9 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -26,6 +26,7 @@ import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.MergeEngineType; +import org.apache.fluss.metadata.RowTtlChangelogMode; import org.apache.fluss.utils.ArrayUtils; import java.lang.reflect.Field; @@ -56,7 +57,9 @@ public class ConfigOptions { public static final String DEFAULT_LISTENER_NAME = "FLUSS"; public static final int KV_FORMAT_VERSION_2 = 2; + public static final int KV_FORMAT_VERSION_3 = 3; public static final int CURRENT_KV_FORMAT_VERSION = KV_FORMAT_VERSION_2; + public static final int MAX_KV_FORMAT_VERSION = KV_FORMAT_VERSION_3; @Internal public static final String[] PARENT_FIRST_LOGGING_PATTERNS = @@ -1560,7 +1563,9 @@ public class ConfigOptions { + "when bucket key differs from primary key, which ensures proper prefix lookup support. " + "When bucket key equals primary key (default bucket key), it still uses datalake's encoder " + "for optimization (encoded bytes can be reused for bucket calculation). " - + "Bucket key encoding always uses datalake's encoder to align with datalake bucket calculation."); + + "Bucket key encoding always uses datalake's encoder to align with datalake bucket calculation. " + + "(3) Version 3: TTL-enabled primary key tables encode a Fluss-owned TTL timestamp " + + "before the row payload so compaction filters can clean up expired rows."); public static final ConfigOption TABLE_KV_STANDBY_REPLICA_ENABLED = key("table.kv.standby-replica.enabled") @@ -1573,6 +1578,33 @@ public class ConfigOptions { + "Tables created before this option was introduced are treated as disabled. " + "Can be dynamically enabled via ALTER TABLE."); + public static final ConfigOption TABLE_KV_ROW_TTL = + key("table.kv.row.ttl") + .durationType() + .noDefaultValue() + .withDescription( + "The best-effort row-level TTL for primary key tables. " + + "If not set, row-level TTL is disabled. " + + "Expired rows may remain visible until RocksDB compaction removes them."); + + public static final ConfigOption TABLE_KV_ROW_TTL_CHANGELOG_MODE = + key("table.kv.row.ttl.changelog-mode") + .enumType(RowTtlChangelogMode.class) + .defaultValue(RowTtlChangelogMode.NONE) + .withDescription( + "The changelog mode for row-level TTL cleanup. " + + "Only 'none' is supported in this version, which means TTL cleanup does not emit delete records."); + + public static final ConfigOption TABLE_KV_ROW_TTL_TIME_COLUMN = + key("table.kv.row.ttl.time-column") + .stringType() + .noDefaultValue() + .withDescription( + "The event-time column for row-level TTL. " + + "If not set, row-level TTL uses processing time. " + + "If set, the column must be BIGINT epoch milliseconds or TIMESTAMP_LTZ. " + + "Rows with null event-time values do not expire through TTL."); + public static final ConfigOption TABLE_AUTO_PARTITION_ENABLED = key("table.auto-partition.enabled") .booleanType() diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index fbf8c77264..beb1390a09 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -25,6 +25,7 @@ import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.MergeEngineType; +import org.apache.fluss.metadata.RowTtlChangelogMode; import org.apache.fluss.utils.AutoPartitionStrategy; import java.time.Duration; @@ -41,6 +42,11 @@ @PublicEvolving public class TableConfig { + /** + * Internal KV table property that binds the row TTL time column to a stable schema column id. + */ + public static final String KV_ROW_TTL_TIME_COLUMN_ID_KEY = "table.kv.row.ttl.time-column-id"; + // the table properties configuration private final Configuration config; @@ -90,6 +96,27 @@ public long getLogTTLMs() { return config.get(ConfigOptions.TABLE_LOG_TTL).toMillis(); } + /** Gets the row-level TTL of the table. */ + public Optional getRowTTL() { + return config.getOptional(ConfigOptions.TABLE_KV_ROW_TTL); + } + + /** Gets the row-level TTL changelog mode of the table. */ + public RowTtlChangelogMode getRowTTLChangelogMode() { + return config.get(ConfigOptions.TABLE_KV_ROW_TTL_CHANGELOG_MODE); + } + + /** Gets the optional row-level TTL time column of the table. */ + public Optional getRowTTLTimeColumn() { + return config.getOptional(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN); + } + + /** Gets the internal row-level TTL time-column id, if event-time TTL is enabled. */ + public Optional getRowTTLTimeColumnId() { + String value = config.toMap().get(KV_ROW_TTL_TIME_COLUMN_ID_KEY); + return value == null ? Optional.empty() : Optional.of(Integer.parseInt(value)); + } + /** Gets the local segments to retain for tiered log of the table. */ public int getTieredLogLocalSegments() { return config.get(ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS); diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/RowTtlChangelogMode.java b/fluss-common/src/main/java/org/apache/fluss/metadata/RowTtlChangelogMode.java new file mode 100644 index 0000000000..644b9565e2 --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/RowTtlChangelogMode.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.annotation.PublicEvolving; + +/** + * Changelog mode for row-level TTL cleanup. + * + * @since 0.10 + */ +@PublicEvolving +public enum RowTtlChangelogMode { + NONE +} diff --git a/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java b/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java index 13906347db..3cb9f41d9a 100644 --- a/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java +++ b/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java @@ -27,10 +27,38 @@ public class BinaryValue { public final short schemaId; public final BinaryRow row; + private final long valueTag; + private final boolean hasValueTag; public BinaryValue(short schemaId, BinaryRow row) { this.schemaId = schemaId; this.row = row; + this.valueTag = 0L; + this.hasValueTag = false; + } + + public BinaryValue(short schemaId, long valueTag, BinaryRow row) { + this.schemaId = schemaId; + this.row = row; + this.valueTag = valueTag; + this.hasValueTag = true; + } + + /** Returns whether this value carries an internal value tag. */ + public boolean hasValueTag() { + return hasValueTag; + } + + /** Returns the internal value tag. */ + public long getValueTag() { + return valueTag; + } + + /** Returns a value with a different row while preserving the value-layout metadata. */ + public BinaryValue withRow(short schemaId, BinaryRow row) { + return hasValueTag + ? new BinaryValue(schemaId, valueTag, row) + : new BinaryValue(schemaId, row); } /** @@ -38,7 +66,9 @@ public BinaryValue(short schemaId, BinaryRow row) { * be expected persisted to kv store. */ public byte[] encodeValue() { - return ValueEncoder.encodeValue(schemaId, row); + return hasValueTag + ? ValueEncoder.encodeValueWithTag(schemaId, valueTag, row) + : ValueEncoder.encodeValue(schemaId, row); } @Override @@ -47,16 +77,28 @@ public boolean equals(Object o) { return false; } BinaryValue that = (BinaryValue) o; - return schemaId == that.schemaId && Objects.equals(row, that.row); + return schemaId == that.schemaId + && valueTag == that.valueTag + && hasValueTag == that.hasValueTag + && Objects.equals(row, that.row); } @Override public int hashCode() { - return Objects.hash(schemaId, row); + return Objects.hash(schemaId, row, valueTag, hasValueTag); } @Override public String toString() { - return "BinaryValue{" + "schemaId=" + schemaId + ", row=" + row + '}'; + return "BinaryValue{" + + "schemaId=" + + schemaId + + ", row=" + + row + + ", valueTag=" + + valueTag + + ", hasValueTag=" + + hasValueTag + + '}'; } } diff --git a/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecord.java b/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecord.java index 8b97825e4e..a3fab28a02 100644 --- a/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecord.java +++ b/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecord.java @@ -21,9 +21,12 @@ import org.apache.fluss.memory.MemorySegmentOutputView; import org.apache.fluss.row.BinaryRow; import org.apache.fluss.row.decode.RowDecoder; +import org.apache.fluss.row.encode.KvValueLayout; import java.io.IOException; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; + /** * A value record is a tuple consisting of a value row and a schema id for the row. * @@ -31,8 +34,8 @@ * *
    *
  • Length => int32 - *
  • SchemaId => int16 - *
  • Value => {@link BinaryRow} + *
  • RawValue => schema id, optional internal fields, and {@link BinaryRow} as defined by {@link + * KvValueLayout} *
* * @since 0.3 @@ -41,16 +44,24 @@ public class DefaultValueRecord implements ValueRecord { static final int LENGTH_OFFSET = 0; static final int LENGTH_LENGTH = 4; - static final int SCHEMA_ID_OFFSET = LENGTH_LENGTH; - static final int SCHEMA_ID_LENGTH = 2; - static final int VALUE_OFFSET = SCHEMA_ID_OFFSET + SCHEMA_ID_LENGTH; private final short schemaId; private final BinaryRow row; + private final int sizeInBytes; public DefaultValueRecord(short schemaId, BinaryRow row) { + this( + schemaId, + row, + LENGTH_LENGTH + + KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_2).rowPayloadOffset() + + row.getSizeInBytes()); + } + + private DefaultValueRecord(short schemaId, BinaryRow row, int sizeInBytes) { this.schemaId = schemaId; this.row = row; + this.sizeInBytes = sizeInBytes; } @Override @@ -65,12 +76,14 @@ public BinaryRow getRow() { @Override public int getSizeInBytes() { - return row.getSizeInBytes() + LENGTH_LENGTH + SCHEMA_ID_LENGTH; + return sizeInBytes; } public int writeTo(MemorySegmentOutputView outputView) throws IOException { - int sizeInBytes = getSizeInBytes(); - outputView.writeInt(sizeInBytes - LENGTH_LENGTH); + KvValueLayout kvValueLayout = KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_2); + int valueLength = kvValueLayout.rowPayloadOffset() + row.getSizeInBytes(); + int sizeInBytes = LENGTH_LENGTH + valueLength; + outputView.writeInt(valueLength); outputView.writeShort(schemaId); outputView.write(row.getSegments()[0], row.getOffset(), row.getSizeInBytes()); return sizeInBytes; @@ -78,14 +91,16 @@ public int writeTo(MemorySegmentOutputView outputView) throws IOException { public static DefaultValueRecord readFrom( MemorySegment segment, int position, ValueRecordBatch.ReadContext readContext) { - int sizeInBytesWithoutLength = segment.getInt(position + LENGTH_OFFSET); - short schemaId = segment.getShort(position + SCHEMA_ID_OFFSET); + int valueLength = segment.getInt(position + LENGTH_OFFSET); + int valueOffset = position + LENGTH_LENGTH; + KvValueLayout kvValueLayout = readContext.getKvValueLayout(); + short schemaId = kvValueLayout.readSchemaId(segment, valueOffset); RowDecoder decoder = readContext.getRowDecoder(schemaId); BinaryRow value = decoder.decode( segment, - position + VALUE_OFFSET, - sizeInBytesWithoutLength - SCHEMA_ID_LENGTH); - return new DefaultValueRecord(schemaId, value); + valueOffset + kvValueLayout.rowPayloadOffset(), + kvValueLayout.rowPayloadLength(valueLength)); + return new DefaultValueRecord(schemaId, value, LENGTH_LENGTH + valueLength); } } diff --git a/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordBatch.java index b4955eb460..03f85ab4af 100644 --- a/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordBatch.java +++ b/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordBatch.java @@ -18,6 +18,7 @@ package org.apache.fluss.record; import org.apache.fluss.row.decode.RowDecoder; +import org.apache.fluss.row.encode.KvValueLayout; /** * A value record batch is a container for a batch of {@link ValueRecord}. @@ -71,5 +72,8 @@ interface ReadContext { * @param schemaId the schema of the kv records */ RowDecoder getRowDecoder(int schemaId); + + /** Gets the KV value layout used to locate row bytes in this value record batch. */ + KvValueLayout getKvValueLayout(); } } diff --git a/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java index e00559e00a..b9e0a35d29 100644 --- a/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java +++ b/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java @@ -21,26 +21,40 @@ import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.SchemaGetter; import org.apache.fluss.row.decode.RowDecoder; +import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.types.DataType; import java.util.HashMap; import java.util.Map; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; + /** A default implementation of {@link ValueRecordBatch.ReadContext} . */ public class ValueRecordReadContext implements ValueRecordBatch.ReadContext { private final Map rowDecoderCache; private final SchemaGetter schemaGetter; private final KvFormat kvFormat; + private final KvValueLayout kvValueLayout; - private ValueRecordReadContext(SchemaGetter schemaGetter, KvFormat kvFormat) { + private ValueRecordReadContext( + SchemaGetter schemaGetter, KvFormat kvFormat, KvValueLayout kvValueLayout) { this.rowDecoderCache = new HashMap<>(); this.schemaGetter = schemaGetter; this.kvFormat = kvFormat; + this.kvValueLayout = kvValueLayout; } + /** Creates a read context for version 2 raw values. */ public static ValueRecordReadContext createReadContext( SchemaGetter schemaGetter, KvFormat kvFormat) { - return new ValueRecordReadContext(schemaGetter, kvFormat); + return createReadContext( + schemaGetter, kvFormat, KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_2)); + } + + /** Creates a read context for the given KV value layout. */ + public static ValueRecordReadContext createReadContext( + SchemaGetter schemaGetter, KvFormat kvFormat, KvValueLayout kvValueLayout) { + return new ValueRecordReadContext(schemaGetter, kvFormat, kvValueLayout); } @Override @@ -53,4 +67,9 @@ public RowDecoder getRowDecoder(int schemaId) { kvFormat, schema.getRowType().getChildren().toArray(new DataType[0])); }); } + + @Override + public KvValueLayout getKvValueLayout() { + return kvValueLayout; + } } diff --git a/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java index d809a095f8..8acbf87497 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java @@ -23,10 +23,11 @@ import org.apache.fluss.record.ValueRecord; import org.apache.fluss.row.InternalRow; import org.apache.fluss.row.ProjectedRow; +import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.types.DataType; import org.apache.fluss.utils.SchemaUtil; -import static org.apache.fluss.row.encode.ValueEncoder.SCHEMA_ID_LENGTH; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; /** * A decoder that deserializes raw byte arrays of {@link ValueRecord} with dynamic or @@ -48,12 +49,28 @@ public class FixedSchemaDecoder { /** Indicates whether there is no projection between source schema and target schema. */ private final boolean noProjection; + /** The raw value layout used to locate the row payload. */ + private final KvValueLayout kvValueLayout; + public FixedSchemaDecoder(KvFormat kvFormat, Schema sourceSchema, Schema targetSchema) { + this( + kvFormat, + sourceSchema, + targetSchema, + KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_2)); + } + + public FixedSchemaDecoder( + KvFormat kvFormat, + Schema sourceSchema, + Schema targetSchema, + KvValueLayout kvValueLayout) { this.rowDecoder = RowDecoder.create( kvFormat, sourceSchema.getRowType().getChildren().toArray(new DataType[0])); this.fieldIdMapping = SchemaUtil.getIndexMapping(sourceSchema, targetSchema); this.noProjection = false; + this.kvValueLayout = kvValueLayout; } /** @@ -61,11 +78,20 @@ public FixedSchemaDecoder(KvFormat kvFormat, Schema sourceSchema, Schema targetS * target schema. */ public FixedSchemaDecoder(KvFormat kvFormat, Schema schema) { + this(kvFormat, schema, KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_2)); + } + + /** + * Creates a FixedSchemaDecoder without projection, i.e., the source schema is the same as the + * target schema. + */ + public FixedSchemaDecoder(KvFormat kvFormat, Schema schema, KvValueLayout kvValueLayout) { this.rowDecoder = RowDecoder.create( kvFormat, schema.getRowType().getChildren().toArray(new DataType[0])); this.fieldIdMapping = null; this.noProjection = true; + this.kvValueLayout = kvValueLayout; } /** @@ -90,6 +116,9 @@ public InternalRow decode(MemorySegment segment, int offset, int sizeInBytes) { * adheres to the fixed {@code targetSchema}. */ public InternalRow decode(MemorySegment valueSegment) { - return decode(valueSegment, SCHEMA_ID_LENGTH, valueSegment.size() - SCHEMA_ID_LENGTH); + return decode( + valueSegment, + kvValueLayout.rowPayloadOffset(), + kvValueLayout.rowPayloadLength(valueSegment.size())); } } diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/KvValueLayout.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/KvValueLayout.java new file mode 100644 index 0000000000..b6019964ad --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/KvValueLayout.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.row.encode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.memory.MemorySegment; +import org.apache.fluss.utils.UnsafeUtils; + +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; +import static org.apache.fluss.config.ConfigOptions.MAX_KV_FORMAT_VERSION; +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkState; + +/** Versioned physical layout of raw KV value bytes. */ +@Internal +public final class KvValueLayout { + + private static final int MIN_KV_FORMAT_VERSION = 1; + private static final int SCHEMA_ID_OFFSET = 0; + private static final int SCHEMA_ID_LENGTH = 2; + private static final int VALUE_TAG_OFFSET = SCHEMA_ID_OFFSET + SCHEMA_ID_LENGTH; + private static final int VALUE_TAG_LENGTH = 8; + + private final boolean hasValueTag; + + private KvValueLayout(boolean hasValueTag) { + this.hasValueTag = hasValueTag; + } + + /** Returns the KV value layout for the table KV format version. */ + public static KvValueLayout forKvFormatVersion(int kvFormatVersion) { + checkArgument( + kvFormatVersion >= MIN_KV_FORMAT_VERSION, + "kvFormatVersion must be at least %s, but was %s.", + MIN_KV_FORMAT_VERSION, + kvFormatVersion); + checkArgument( + kvFormatVersion <= MAX_KV_FORMAT_VERSION, + "kvFormatVersion must not exceed %s, but was %s.", + MAX_KV_FORMAT_VERSION, + kvFormatVersion); + return new KvValueLayout(kvFormatVersion >= KV_FORMAT_VERSION_3); + } + + /** Returns the byte offset of schema id in a raw KV value. */ + public int schemaIdOffset() { + return SCHEMA_ID_OFFSET; + } + + /** Returns the encoded schema id length in bytes. */ + public int schemaIdLength() { + return SCHEMA_ID_LENGTH; + } + + /** Returns whether the raw KV value has an internal value tag. */ + public boolean hasValueTag() { + return hasValueTag; + } + + /** Returns the byte offset of the internal value tag. */ + public int valueTagOffset() { + checkState(hasValueTag, "KV value layout does not have a value tag."); + return VALUE_TAG_OFFSET; + } + + /** Returns the internal value tag length in bytes. */ + public int valueTagLength() { + return hasValueTag ? VALUE_TAG_LENGTH : 0; + } + + /** Returns the byte offset of row payload in a raw KV value. */ + public int rowPayloadOffset() { + return SCHEMA_ID_OFFSET + SCHEMA_ID_LENGTH + valueTagLength(); + } + + /** Returns the row payload length for a raw KV value length. */ + public int rowPayloadLength(int valueLength) { + int rowPayloadOffset = rowPayloadOffset(); + checkArgument( + valueLength >= rowPayloadOffset, + "valueLength must be at least row payload offset %s, but was %s.", + rowPayloadOffset, + valueLength); + return valueLength - rowPayloadOffset; + } + + /** Reads the schema id from a raw KV value. */ + public short readSchemaId(MemorySegment value) { + return readSchemaId(value, 0); + } + + /** Reads the schema id from a raw KV value embedded at the given offset. */ + public short readSchemaId(MemorySegment value, int valueOffset) { + return value.getShort(valueOffset + schemaIdOffset()); + } + + /** Writes the schema id to a raw KV value. */ + public void writeSchemaId(byte[] value, short schemaId) { + writeSchemaId(value, 0, schemaId); + } + + /** Writes the schema id to a raw KV value embedded at the given offset. */ + public void writeSchemaId(byte[] value, int valueOffset, short schemaId) { + UnsafeUtils.putShort(value, valueOffset + schemaIdOffset(), schemaId); + } + + /** Reads the internal value tag from a raw KV value. */ + public long readValueTag(MemorySegment value) { + return readValueTag(value, 0); + } + + /** Reads the internal value tag from a raw KV value embedded at the given offset. */ + public long readValueTag(MemorySegment value, int valueOffset) { + return value.getLongBigEndian(valueOffset + valueTagOffset()); + } + + /** Writes the internal value tag to a raw KV value. */ + public void writeValueTag(byte[] value, long valueTag) { + writeValueTag(value, 0, valueTag); + } + + /** Writes the internal value tag to a raw KV value embedded at the given offset. */ + public void writeValueTag(byte[] value, int valueOffset, long valueTag) { + MemorySegment.wrap(value).putLongBigEndian(valueOffset + valueTagOffset(), valueTag); + } +} diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java index 5e243799cc..bd5e0cc5f2 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java @@ -29,7 +29,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static org.apache.fluss.row.encode.ValueEncoder.SCHEMA_ID_LENGTH; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; /** * A decoder to decode a schema id and {@link BinaryRow} from a byte array value which is encoded by @@ -40,17 +40,27 @@ public class ValueDecoder { private final Map rowDecoders; private final SchemaGetter schemaGetter; private final KvFormat kvFormat; + private final KvValueLayout kvValueLayout; public ValueDecoder(SchemaGetter schemaGetter, KvFormat kvFormat) { + this(schemaGetter, kvFormat, KV_FORMAT_VERSION_2); + } + + public ValueDecoder(SchemaGetter schemaGetter, KvFormat kvFormat, int kvFormatVersion) { + this(schemaGetter, kvFormat, KvValueLayout.forKvFormatVersion(kvFormatVersion)); + } + + public ValueDecoder(SchemaGetter schemaGetter, KvFormat kvFormat, KvValueLayout kvValueLayout) { this.rowDecoders = new ConcurrentHashMap<>(); this.schemaGetter = schemaGetter; this.kvFormat = kvFormat; + this.kvValueLayout = kvValueLayout; } /** Decode the value bytes and return the schema id and the row encoded in the value bytes. */ public BinaryValue decodeValue(byte[] valueBytes) { MemorySegment memorySegment = MemorySegment.wrap(valueBytes); - short schemaId = memorySegment.getShort(0); + short schemaId = kvValueLayout.readSchemaId(memorySegment); RowDecoder rowDecoder = rowDecoders.computeIfAbsent( @@ -64,7 +74,12 @@ public BinaryValue decodeValue(byte[] valueBytes) { BinaryRow row = rowDecoder.decode( - memorySegment, SCHEMA_ID_LENGTH, valueBytes.length - SCHEMA_ID_LENGTH); + memorySegment, + kvValueLayout.rowPayloadOffset(), + kvValueLayout.rowPayloadLength(valueBytes.length)); + if (kvValueLayout.hasValueTag()) { + return new BinaryValue(schemaId, kvValueLayout.readValueTag(memorySegment), row); + } return new BinaryValue(schemaId, row); } } diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java index 2d842ab173..7c856fd8c8 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java @@ -17,13 +17,72 @@ package org.apache.fluss.row.encode; +import org.apache.fluss.record.BinaryValue; import org.apache.fluss.row.BinaryRow; -import org.apache.fluss.utils.UnsafeUtils; + +import javax.annotation.Nullable; + +import java.util.function.ToLongFunction; + +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** An encoder to encode {@link BinaryRow} with a schema id as value to be stored in kv store. */ public class ValueEncoder { - public static final int SCHEMA_ID_LENGTH = 2; + private final KvValueLayout kvValueLayout; + @Nullable private final ToLongFunction valueTagProvider; + + private ValueEncoder( + KvValueLayout kvValueLayout, @Nullable ToLongFunction valueTagProvider) { + this.kvValueLayout = kvValueLayout; + this.valueTagProvider = valueTagProvider; + } + + /** Creates a version-aware value encoder for the table KV format version. */ + public static ValueEncoder forKvFormatVersion( + int kvFormatVersion, @Nullable ToLongFunction valueTagProvider) { + KvValueLayout kvValueLayout = KvValueLayout.forKvFormatVersion(kvFormatVersion); + if (kvValueLayout.hasValueTag() && valueTagProvider == null) { + throw new IllegalArgumentException( + "valueTagProvider must be non-null for a KV value layout with a value tag."); + } + if (!kvValueLayout.hasValueTag() && valueTagProvider != null) { + throw new IllegalArgumentException( + "valueTagProvider must be null for a KV value layout without a value tag."); + } + return new ValueEncoder(kvValueLayout, valueTagProvider); + } + + /** + * Creates a value encoder with the same KV format version and a different value tag provider. + */ + public ValueEncoder withValueTagProvider(ToLongFunction valueTagProvider) { + if (!kvValueLayout.hasValueTag()) { + throw new IllegalStateException( + "valueTagProvider can only be replaced for a KV value layout with a value tag."); + } + checkNotNull(valueTagProvider, "valueTagProvider must not be null."); + return new ValueEncoder(kvValueLayout, valueTagProvider); + } + + /** Returns whether this encoder writes an internal value tag before the row bytes. */ + public boolean hasValueTag() { + return kvValueLayout.hasValueTag(); + } + + /** Creates a binary value using the table KV format version bound to this encoder. */ + public BinaryValue createValue(short schemaId, BinaryRow row) { + if (kvValueLayout.hasValueTag()) { + return new BinaryValue( + schemaId, + checkNotNull(valueTagProvider, "valueTagProvider must not be null.") + .applyAsLong(row), + row); + } + return new BinaryValue(schemaId, row); + } /** * Encode the {@code row} with a {@code schemaId} to a byte array value to be expected persisted @@ -33,9 +92,27 @@ public class ValueEncoder { * @param row the row to encode */ public static byte[] encodeValue(short schemaId, BinaryRow row) { - byte[] values = new byte[SCHEMA_ID_LENGTH + row.getSizeInBytes()]; - UnsafeUtils.putShort(values, 0, schemaId); - row.copyTo(values, SCHEMA_ID_LENGTH); + KvValueLayout kvValueLayout = KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_2); + byte[] values = new byte[kvValueLayout.rowPayloadOffset() + row.getSizeInBytes()]; + kvValueLayout.writeSchemaId(values, schemaId); + row.copyTo(values, kvValueLayout.rowPayloadOffset()); + return values; + } + + /** + * Encode the {@code row} with a {@code schemaId} and value tag to a byte array value to be + * expected persisted to kv store. + * + * @param schemaId the schema id of the row + * @param valueTag the opaque value tag + * @param row the row to encode + */ + public static byte[] encodeValueWithTag(short schemaId, long valueTag, BinaryRow row) { + KvValueLayout kvValueLayout = KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_3); + byte[] values = new byte[kvValueLayout.rowPayloadOffset() + row.getSizeInBytes()]; + kvValueLayout.writeSchemaId(values, schemaId); + kvValueLayout.writeValueTag(values, valueTag); + row.copyTo(values, kvValueLayout.rowPayloadOffset()); return values; } } diff --git a/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java b/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java new file mode 100644 index 0000000000..6263ce490d --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/row/encode/KvValueLayoutTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.row.encode; + +import org.apache.fluss.memory.MemorySegment; +import org.apache.fluss.metadata.KvFormat; +import org.apache.fluss.record.BinaryValue; +import org.apache.fluss.record.DefaultValueRecordBatch; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.record.ValueRecord; +import org.apache.fluss.record.ValueRecordReadContext; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.row.decode.FixedSchemaDecoder; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for the versioned KV value layout. */ +class KvValueLayoutTest { + + @Test + void testVersion3KvValueLayoutStoresBigEndianValueTag() { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + long valueTag = 1234567890123L; + KvValueLayout kvValueLayout = KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_3); + + BinaryValue value = + ValueEncoder.forKvFormatVersion(KV_FORMAT_VERSION_3, ignored -> valueTag) + .createValue(DEFAULT_SCHEMA_ID, row); + byte[] encoded = value.encodeValue(); + MemorySegment segment = MemorySegment.wrap(encoded); + + assertThat(encoded).hasSize(kvValueLayout.rowPayloadOffset() + row.getSizeInBytes()); + assertThat(kvValueLayout.readSchemaId(segment)).isEqualTo(DEFAULT_SCHEMA_ID); + assertThat(kvValueLayout.readValueTag(segment)).isEqualTo(valueTag); + assertThat(value.hasValueTag()).isTrue(); + assertThat(value.getValueTag()).isEqualTo(valueTag); + + byte[] expectedRowBytes = new byte[row.getSizeInBytes()]; + row.copyTo(expectedRowBytes, 0); + assertThat(Arrays.copyOfRange(encoded, kvValueLayout.rowPayloadOffset(), encoded.length)) + .isEqualTo(expectedRowBytes); + + BinaryValue decoded = + new ValueDecoder( + new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA1_SCHEMA), + KvFormat.COMPACTED, + KV_FORMAT_VERSION_3) + .decodeValue(encoded); + assertThat(decoded.schemaId).isEqualTo(DEFAULT_SCHEMA_ID); + assertThat(decoded.getValueTag()).isEqualTo(valueTag); + assertThat(decoded.row.getInt(0)).isEqualTo(1); + assertThat(decoded.row.getString(1).toString()).isEqualTo("a"); + } + + @Test + void testVersion3EncoderCanOverrideValueTagProvider() { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + ValueEncoder writeEncoder = + ValueEncoder.forKvFormatVersion(KV_FORMAT_VERSION_3, ignored -> 100L); + + BinaryValue recoveredValue = + writeEncoder + .withValueTagProvider(ignored -> 200L) + .createValue(DEFAULT_SCHEMA_ID, row); + + assertThat(recoveredValue.getValueTag()).isEqualTo(200L); + assertThat(writeEncoder.createValue(DEFAULT_SCHEMA_ID, row).getValueTag()).isEqualTo(100L); + } + + @Test + void testVersion3ValueRecordBatchDecodesThroughKvValueLayout() throws Exception { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + byte[] encodedValue = ValueEncoder.encodeValueWithTag(DEFAULT_SCHEMA_ID, 100L, row); + DefaultValueRecordBatch.Builder builder = DefaultValueRecordBatch.builder(); + builder.append(encodedValue); + DefaultValueRecordBatch recordBatch = builder.build(); + + ValueRecord valueRecord = + recordBatch + .records( + ValueRecordReadContext.createReadContext( + new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA1_SCHEMA), + KvFormat.COMPACTED, + KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_3))) + .iterator() + .next(); + + assertThat(valueRecord.schemaId()).isEqualTo(DEFAULT_SCHEMA_ID); + assertThat(valueRecord.getRow().getInt(0)).isEqualTo(1); + assertThat(valueRecord.getRow().getString(1).toString()).isEqualTo("a"); + } + + @Test + void testFixedSchemaDecoderDecodesVersion3ValueThroughKvValueLayout() { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + byte[] encodedValue = ValueEncoder.encodeValueWithTag(DEFAULT_SCHEMA_ID, 100L, row); + FixedSchemaDecoder decoder = + new FixedSchemaDecoder( + KvFormat.COMPACTED, + DATA1_SCHEMA, + KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_3)); + + InternalRow decoded = decoder.decode(MemorySegment.wrap(encodedValue)); + + assertThat(decoded.getInt(0)).isEqualTo(1); + assertThat(decoded.getString(1).toString()).isEqualTo("a"); + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java index 24f3d94d67..51c1828e08 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java @@ -832,7 +832,7 @@ public boolean applyAggregates( List groupingSets, List aggregateExpressions, DataType dataType) { - // Only supports 'select count(*)/count(1) from source' for log table now. + // Only supports global count when an exact row count is available. if (streaming || aggregateExpressions.size() != 1 || groupingSets.size() > 1 @@ -840,7 +840,8 @@ public boolean applyAggregates( // The count pushdown feature is not supported when the data lake is enabled. // Otherwise, it'll cause miss count data in lake. But In the future, we can push // down count into lake. - || isDataLakeEnabled) { + || isDataLakeEnabled + || !canPushDownRowCount()) { return false; } @@ -876,6 +877,14 @@ public boolean applyAggregates( return true; } + private boolean canPushDownRowCount() { + if (!hasPrimaryKey()) { + return true; + } + return tableConfig.getChangelogImage() != ChangelogImage.WAL + && !tableConfig.getRowTTL().isPresent(); + } + private Map getPrimaryKeyTypes() { Map pkTypes = new HashMap<>(); for (int index : primaryKeyIndexes) { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java index 01f9a9817e..f22a1113f8 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java @@ -351,10 +351,12 @@ public static org.apache.flink.configuration.ConfigOption toFlinkOption( } else if (clazz.equals(Duration.class)) { // use string type in Flink option instead to make convert back easier option = - builder.stringType() - .defaultValue( - TimeUtils.formatWithHighestUnit( - (Duration) flussOption.defaultValue())); + flussOption.hasDefaultValue() + ? builder.stringType() + .defaultValue( + TimeUtils.formatWithHighestUnit( + (Duration) flussOption.defaultValue())) + : builder.stringType().noDefaultValue(); } else if (clazz.equals(Password.class)) { String defaultValue = ((Password) flussOption.defaultValue()).value(); option = builder.stringType().defaultValue(defaultValue); diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java index 51d05fa036..9cfc8062da 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java @@ -20,7 +20,6 @@ import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.writer.AppendWriter; import org.apache.fluss.client.table.writer.UpsertWriter; -import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.flink.utils.FlinkTestBase; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.row.BinaryString; @@ -456,12 +455,34 @@ void testCountPushDownWithWALMode() throws Exception { + " with ('bucket.num' = '4', 'table.changelog.image' = 'wal')", tableName)) .await(); - // normal scan + String query = String.format("SELECT COUNT(*) FROM %s", tableName); - assertThatThrownBy(() -> tEnv.executeSql(query)) - .hasRootCauseInstanceOf(InvalidTableException.class) + assertThatThrownBy(() -> tEnv.explainSql(query)) .hasMessageContaining( - "Row count is disabled for this table 'defaultdb.test_count_table_with_wal'."); + "Currently, Fluss only support queries on table with datalake enabled or point queries on primary key when it's in batch execution mode."); + } + + @Test + void testCountPushDownWithRowTTL() throws Exception { + String tableName = "test_count_table_with_row_ttl"; + tEnv.executeSql( + String.format( + "create table %s (" + + " id int not null," + + " address varchar," + + " name varchar," + + " primary key (id) NOT ENFORCED)" + + " with (" + + " 'bucket.num' = '4'," + + " 'table.kv.row.ttl' = '1 h'," + + " 'table.kv.format-version' = '3')", + tableName)) + .await(); + + String query = String.format("SELECT COUNT(*) FROM %s", tableName); + assertThatThrownBy(() -> tEnv.explainSql(query)) + .hasMessageContaining( + "Currently, Fluss only support queries on table with datalake enabled or point queries on primary key when it's in batch execution mode."); } @ParameterizedTest diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java index efc386eb64..e70a6e967b 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java @@ -383,6 +383,15 @@ void testOptionConversions() { .withDescription( ConfigOptions.CLIENT_WRITER_BUFFER_MEMORY_SIZE .description())); + + flinkOption = FlinkConversions.toFlinkOption(ConfigOptions.TABLE_KV_ROW_TTL); + assertThat(flinkOption) + .isEqualTo( + org.apache.flink.configuration.ConfigOptions.key( + ConfigOptions.TABLE_KV_ROW_TTL.key()) + .stringType() + .noDefaultValue() + .withDescription(ConfigOptions.TABLE_KV_ROW_TTL.description())); } @Test diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java index b03e1ec63a..7274211d02 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java @@ -48,20 +48,22 @@ public enum ApiKeys { // Version 0: Uses lake's encoder for primary key encoding (legacy behavior). // Version 1: Uses CompactedKeyEncoder for primary key encoding when bucket key differs from // primary key, enabling prefix lookup support. - PUT_KV(1016, 0, 1, PUBLIC), + // Version 2: Requires row-TTL-aware clients for row TTL primary key tables. + PUT_KV(1016, 0, 2, PUBLIC), // Version 0: Uses lake's encoder for primary key encoding (legacy behavior). // Version 1: Uses CompactedKeyEncoder for primary key encoding when bucket key differs from // primary key, enabling prefix lookup support. - LOOKUP(1017, 0, 1, PUBLIC), + // Version 2: Requires row-TTL-aware clients for row TTL primary key tables. + LOOKUP(1017, 0, 2, PUBLIC), NOTIFY_LEADER_AND_ISR(1018, 0, 0, PRIVATE), STOP_REPLICA(1019, 0, 0, PRIVATE), ADJUST_ISR(1020, 0, 0, PRIVATE), LIST_OFFSETS(1021, 0, 0, PUBLIC), COMMIT_KV_SNAPSHOT(1022, 0, 0, PRIVATE), - GET_LATEST_KV_SNAPSHOTS(1023, 0, 0, PUBLIC), - GET_KV_SNAPSHOT_METADATA(1024, 0, 0, PUBLIC), + GET_LATEST_KV_SNAPSHOTS(1023, 0, 1, PUBLIC), + GET_KV_SNAPSHOT_METADATA(1024, 0, 1, PUBLIC), GET_FILESYSTEM_SECURITY_TOKEN(1025, 0, 0, PUBLIC), INIT_WRITER(1026, 0, 0, PUBLIC), COMMIT_REMOTE_LOG_MANIFEST(1027, 0, 0, PRIVATE), @@ -70,12 +72,15 @@ public enum ApiKeys { COMMIT_LAKE_TABLE_SNAPSHOT(1030, 0, 0, PRIVATE), NOTIFY_LAKE_TABLE_OFFSET(1031, 0, 0, PRIVATE), GET_LAKE_SNAPSHOT(1032, 0, 0, PUBLIC), - LIMIT_SCAN(1033, 0, 0, PUBLIC), + + // Version 1: Requires row-TTL-aware clients for row TTL primary key tables. + LIMIT_SCAN(1033, 0, 1, PUBLIC), // Version 0: Uses lake's encoder for prefix key encoding (legacy behavior). // Version 1: Uses CompactedKeyEncoder for prefix key encoding when bucket key differs from // primary key, ensuring encoded bucket key bytes are a prefix of primary key bytes. - PREFIX_LOOKUP(1034, 0, 1, PUBLIC), + // Version 2: Requires row-TTL-aware clients for row TTL primary key tables. + PREFIX_LOOKUP(1034, 0, 2, PUBLIC), GET_DATABASE_INFO(1035, 0, 0, PUBLIC), CREATE_PARTITION(1036, 0, 0, PUBLIC), @@ -98,12 +103,12 @@ public enum ApiKeys { REGISTER_PRODUCER_OFFSETS(1053, 0, 0, PUBLIC), GET_PRODUCER_OFFSETS(1054, 0, 0, PUBLIC), DELETE_PRODUCER_OFFSETS(1055, 0, 0, PUBLIC), - ACQUIRE_KV_SNAPSHOT_LEASE(1056, 0, 0, PUBLIC), + ACQUIRE_KV_SNAPSHOT_LEASE(1056, 0, 1, PUBLIC), RELEASE_KV_SNAPSHOT_LEASE(1057, 0, 0, PUBLIC), DROP_KV_SNAPSHOT_LEASE(1058, 0, 0, PUBLIC), GET_TABLE_STATS(1059, 0, 0, PUBLIC), ALTER_DATABASE(1060, 0, 0, PUBLIC), - SCAN_KV(1061, 0, 0, PUBLIC), + SCAN_KV(1061, 0, 1, PUBLIC), GET_CLUSTER_HEALTH(1062, 0, 0, PUBLIC), LIST_REMOTE_LOG_MANIFESTS(1063, 0, 0, PUBLIC), LIST_KV_SNAPSHOTS(1064, 0, 0, PUBLIC); diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/protocol/ApiKeysTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/protocol/ApiKeysTest.java index 0ef7ad22ed..e40f47fafe 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/protocol/ApiKeysTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/protocol/ApiKeysTest.java @@ -38,4 +38,20 @@ void testAllApiKeys() { assertThat(api.highestSupportedVersion).isGreaterThanOrEqualTo((short) 0); } } + + @Test + void testSnapshotAndScanApisUseVersionOneForRowTTLAwareClients() { + assertThat(ApiKeys.GET_LATEST_KV_SNAPSHOTS.highestSupportedVersion).isEqualTo((short) 1); + assertThat(ApiKeys.GET_KV_SNAPSHOT_METADATA.highestSupportedVersion).isEqualTo((short) 1); + assertThat(ApiKeys.ACQUIRE_KV_SNAPSHOT_LEASE.highestSupportedVersion).isEqualTo((short) 1); + assertThat(ApiKeys.SCAN_KV.highestSupportedVersion).isEqualTo((short) 1); + assertThat(ApiKeys.LIMIT_SCAN.highestSupportedVersion).isEqualTo((short) 1); + } + + @Test + void testRawKvApisUseVersionTwoForRowTTLAwareClients() { + assertThat(ApiKeys.PUT_KV.highestSupportedVersion).isEqualTo((short) 2); + assertThat(ApiKeys.LOOKUP.highestSupportedVersion).isEqualTo((short) 2); + assertThat(ApiKeys.PREFIX_LOOKUP.highestSupportedVersion).isEqualTo((short) 2); + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java index f7b6de9eb8..809b64eb72 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java @@ -88,6 +88,7 @@ import org.apache.fluss.server.metadata.PartitionMetadata; import org.apache.fluss.server.metadata.ServerMetadataCache; import org.apache.fluss.server.metadata.TableMetadata; +import org.apache.fluss.server.replica.ReplicaManager; import org.apache.fluss.server.tablet.TabletService; import org.apache.fluss.server.utils.ServerRpcMessageUtils; import org.apache.fluss.server.zk.ZooKeeperClient; @@ -174,6 +175,9 @@ public ServerType providerType() { public abstract void authorizeTable(OperationType operationType, long tableId); + /** Returns table information for a table id known by the concrete server role. */ + protected abstract TableInfo getTableInfo(long tableId); + public void authorizeDatabase(OperationType operationType, String databaseName) { if (authorizer != null) { authorizer.authorize(currentSession(), operationType, Resource.database(databaseName)); @@ -372,6 +376,7 @@ public CompletableFuture getLatestKvSnapshots( + tablePath + "' is a partitioned table, but partition name is not provided."); } + validateClientVersionForPkTable(ApiKeys.GET_LATEST_KV_SNAPSHOTS, tableInfo); try { // get table id @@ -415,6 +420,7 @@ public CompletableFuture getKvSnapshotMetadata( GetKvSnapshotMetadataRequest request) { long tableId = request.getTableId(); authorizeTable(OperationType.DESCRIBE, tableId); + validateClientVersionForPkTable(ApiKeys.GET_KV_SNAPSHOT_METADATA, tableId); TableBucket tableBucket = new TableBucket( @@ -439,6 +445,21 @@ public CompletableFuture getKvSnapshotMetadata( } } + /** Validates whether the current client API version can access this primary-key table. */ + protected void validateClientVersionForPkTable(ApiKeys apiKey, TableInfo tableInfo) { + ReplicaManager.validateClientVersionForPkTable( + apiKey, currentSession().getApiVersion(), tableInfo); + } + + /** Validates current API version for a table id, avoiding metadata lookup when safe. */ + protected void validateClientVersionForPkTable(ApiKeys apiKey, long tableId) { + if (ReplicaManager.canSkipClientVersionValidation( + apiKey, currentSession().getApiVersion())) { + return; + } + validateClientVersionForPkTable(apiKey, getTableInfo(tableId)); + } + @Override public CompletableFuture getFileSystemSecurityToken( GetFileSystemSecurityTokenRequest request) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index 0772f5e3db..40c58f2613 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -24,6 +24,7 @@ import org.apache.fluss.cluster.rebalance.ServerTag; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; import org.apache.fluss.config.cluster.AlterConfig; import org.apache.fluss.config.cluster.AlterConfigOpType; import org.apache.fluss.exception.ApiException; @@ -52,6 +53,7 @@ import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.PartitionSpec; import org.apache.fluss.metadata.ResolvedPartitionSpec; +import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -135,6 +137,7 @@ import org.apache.fluss.rpc.messages.RemoveServerTagResponse; import org.apache.fluss.rpc.netty.server.Session; import org.apache.fluss.rpc.protocol.ApiError; +import org.apache.fluss.rpc.protocol.ApiKeys; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.security.acl.AclBinding; import org.apache.fluss.security.acl.AclBindingFilter; @@ -208,6 +211,8 @@ import java.util.stream.Collectors; import static org.apache.fluss.config.ConfigOptions.CURRENT_KV_FORMAT_VERSION; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; +import static org.apache.fluss.config.ConfigOptions.MAX_KV_FORMAT_VERSION; import static org.apache.fluss.config.FlussConfigUtils.isTableStorageConfig; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toAclBindingFilters; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toAclBindings; @@ -366,13 +371,20 @@ private TablePath getTablePathById(long tableId) { return tablePath; } - private void validateKvTable(long tableId) { + @Override + protected TableInfo getTableInfo(long tableId) { + TablePath tablePath = getTablePathById(tableId); + return metadataManager.getTable(tablePath); + } + + private TableInfo validateKvTable(long tableId) { TablePath tablePath = getTablePathById(tableId); TableInfo tableInfo = metadataManager.getTable(tablePath); if (!tableInfo.hasPrimaryKey()) { throw new NonPrimaryKeyTableException( "Table '" + tablePath + "' is not a primary key table"); } + return tableInfo; } @Override @@ -664,20 +676,23 @@ private TableDescriptor applySystemDefaults( if (newDescriptor.hasPrimaryKey()) { Map newProperties = new HashMap<>(newDescriptor.getProperties()); - Integer formatVersion = - Configuration.fromMap(newProperties).get(ConfigOptions.TABLE_KV_FORMAT_VERSION); + Configuration newTableConf = Configuration.fromMap(newProperties); + Integer formatVersion = newTableConf.get(ConfigOptions.TABLE_KV_FORMAT_VERSION); if (formatVersion == null) { - // set current kv format version for default + int defaultKvFormatVersion = + newTableConf.getOptional(ConfigOptions.TABLE_KV_ROW_TTL).isPresent() + ? KV_FORMAT_VERSION_3 + : CURRENT_KV_FORMAT_VERSION; newProperties.put( ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), - String.valueOf(CURRENT_KV_FORMAT_VERSION)); + String.valueOf(defaultKvFormatVersion)); } else { - if (formatVersion > CURRENT_KV_FORMAT_VERSION) { + if (formatVersion > MAX_KV_FORMAT_VERSION) { throw new InvalidConfigException( String.format( "Unsupported kv format version %d. " + "The maximum supported version is %d.", - formatVersion, CURRENT_KV_FORMAT_VERSION)); + formatVersion, MAX_KV_FORMAT_VERSION)); } } @@ -686,12 +701,33 @@ private TableDescriptor applySystemDefaults( newProperties.put(ConfigOptions.TABLE_KV_STANDBY_REPLICA_ENABLED.key(), "true"); } + Optional rowTtlTimeColumn = + newTableConf.getOptional(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN); + if (rowTtlTimeColumn.isPresent()) { + Optional rowTtlTimeColumnId = + getColumnId(newDescriptor.getSchema(), rowTtlTimeColumn.get()); + if (rowTtlTimeColumnId.isPresent()) { + newProperties.put( + TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY, + String.valueOf(rowTtlTimeColumnId.get())); + } + } + newDescriptor = newDescriptor.withProperties(newProperties); } return newDescriptor; } + private static Optional getColumnId(Schema schema, String columnName) { + for (Schema.Column column : schema.getColumns()) { + if (column.getName().equals(columnName)) { + return Optional.of(column.getColumnId()); + } + } + return Optional.empty(); + } + private boolean isDataLakeEnabled(TableDescriptor tableDescriptor) { String dataLakeEnabledValue = tableDescriptor.getProperties().get(ConfigOptions.TABLE_DATALAKE_ENABLED.key()); @@ -1186,7 +1222,8 @@ public CompletableFuture acquireKvSnapshotLease( authorizeTable(OperationType.READ, tableId); } - validateKvTable(tableId); + TableInfo tableInfo = validateKvTable(tableId); + validateClientVersionForPkTable(ApiKeys.ACQUIRE_KV_SNAPSHOT_LEASE, tableInfo); } String leaseId = request.getLeaseId(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java index 9f2f1f426f..7a2c69c27d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java @@ -48,6 +48,8 @@ import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocatorUtil; import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; +import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.types.Tuple2; import org.rocksdb.RateLimiter; @@ -117,6 +119,8 @@ public static RateLimiter getDefaultRateLimiter() { private final ZooKeeperClient zkClient; + private final Clock clock; + private final Map currentKvs = new ConcurrentHashMap<>(); /** @@ -148,7 +152,8 @@ private KvManager( ZooKeeperClient zkClient, int recoveryThreadsPerDataDir, LogManager logManager, - TabletServerMetricGroup tabletServerMetricGroup) + TabletServerMetricGroup tabletServerMetricGroup, + Clock clock) throws IOException { super(TabletType.KV, localDiskManager.dataDirs(), conf, recoveryThreadsPerDataDir); this.localDiskManager = localDiskManager; @@ -156,6 +161,7 @@ private KvManager( this.arrowBufferAllocator = BufferAllocatorUtil.createBufferAllocator(null); this.memorySegmentPool = LazyMemorySegmentPool.createServerBufferPool(conf); this.zkClient = zkClient; + this.clock = clock; this.remoteKvDir = FlussPaths.remoteKvDir(conf); this.remoteFileSystem = remoteKvDir.getFileSystem(); this.serverMetricGroup = tabletServerMetricGroup; @@ -189,13 +195,31 @@ public static KvManager create( TabletServerMetricGroup tabletServerMetricGroup, LocalDiskManager localDiskManager) throws IOException { + return create( + conf, + zkClient, + logManager, + tabletServerMetricGroup, + localDiskManager, + SystemClock.getInstance()); + } + + public static KvManager create( + Configuration conf, + ZooKeeperClient zkClient, + LogManager logManager, + TabletServerMetricGroup tabletServerMetricGroup, + LocalDiskManager localDiskManager, + Clock clock) + throws IOException { return new KvManager( localDiskManager, conf, zkClient, conf.getInt(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS), logManager, - tabletServerMetricGroup); + tabletServerMetricGroup, + clock); } public void startup() { @@ -276,7 +300,10 @@ public KvTablet getOrCreateKv( schemaGetter, tableConfig.getChangelogImage(), sharedRocksDBRateLimiter, - autoIncrementManager); + autoIncrementManager, + clock, + getKvFormatVersion(tableConfig), + tableConfig); currentKvs.put(tableBucket, tablet); LOG.info( @@ -368,7 +395,6 @@ public KvTablet loadKv(File tabletDir, SchemaGetter schemaGetter) throws Excepti // TODO: we should support recover schema from disk to decouple put and schema. TablePath tablePath = physicalTablePath.getTablePath(); TableInfo tableInfo = getTableInfo(zkClient, tablePath); - TableConfig tableConfig = tableInfo.getTableConfig(); RowMerger rowMerger = RowMerger.create(tableConfig, tableConfig.getKvFormat(), schemaGetter); @@ -394,7 +420,10 @@ public KvTablet loadKv(File tabletDir, SchemaGetter schemaGetter) throws Excepti schemaGetter, tableConfig.getChangelogImage(), sharedRocksDBRateLimiter, - autoIncrementManager); + autoIncrementManager, + clock, + getKvFormatVersion(tableConfig), + tableConfig); if (this.currentKvs.containsKey(tableBucket)) { throw new IllegalStateException( String.format( @@ -412,6 +441,10 @@ public KvTablet loadKv(File tabletDir, SchemaGetter schemaGetter) throws Excepti return kvTablet; } + private static int getKvFormatVersion(TableConfig tableConfig) { + return tableConfig.getKvFormatVersion().orElse(ConfigOptions.KV_FORMAT_VERSION_2); + } + public void deleteRemoteKvSnapshot( PhysicalTablePath physicalTablePath, TableBucket tableBucket) { FsPath remoteKvTabletDir = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java index f78a258ecb..0c0643bf0a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java @@ -17,12 +17,15 @@ package org.apache.fluss.server.kv; +import org.apache.fluss.annotation.VisibleForTesting; +import org.apache.fluss.config.TableConfig; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.SchemaGetter; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.BinaryValue; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.LogRecord; import org.apache.fluss.record.LogRecordBatch; @@ -74,6 +77,9 @@ public class KvRecoverHelper { private KeyEncoder keyEncoder; private RowEncoder rowEncoder; + private final ValueEncoder valueEncoder; + private final ValueEncoder recoveryValueEncoder; + @Nullable private final RowTtlTimestampProvider recoveryTimestampProvider; private final SchemaGetter schemaGetter; private InternalRow.FieldGetter[] currentFieldGetters; @@ -87,6 +93,7 @@ public KvRecoverHelper( KvRecoverContext recoverContext, KvFormat kvFormat, LogFormat logFormat, + TableConfig tableConfig, SchemaGetter schemaGetter, RemoteLogFetcher remoteLogFetcher) { this.kvTablet = kvTablet; @@ -98,6 +105,15 @@ public KvRecoverHelper( this.kvFormat = kvFormat; this.logFormat = logFormat; this.schemaGetter = schemaGetter; + this.valueEncoder = kvTablet.getValueEncoder(); + this.recoveryTimestampProvider = + valueEncoder.hasValueTag() + ? RowTtlTimestampProvider.forRecovery(tableConfig, schemaGetter) + : null; + this.recoveryValueEncoder = + recoveryTimestampProvider == null + ? valueEncoder + : valueEncoder.withValueTagProvider(recoveryTimestampProvider); this.remoteLogFetcher = remoteLogFetcher; } @@ -268,7 +284,14 @@ private long applyLogRecordBatch( // the log row format may not compatible with kv row format, // e.g, arrow vs. compacted, thus needs a conversion here. BinaryRow row = toKvRow(logRow); - value = ValueEncoder.encodeValue(currentSchemaId.shortValue(), row); + value = + createRecoveredValue( + recoveryValueEncoder, + recoveryTimestampProvider, + currentSchemaId.shortValue(), + row, + logRecord.timestamp()) + .encodeValue(); } resumeRecordConsumer.accept( new KeyValueAndLogOffset( @@ -313,6 +336,19 @@ private BinaryRow toKvRow(InternalRow originalRow) { return rowEncoder.finishRow(); } + @VisibleForTesting + static BinaryValue createRecoveredValue( + ValueEncoder valueEncoder, + @Nullable RowTtlTimestampProvider timestampProvider, + short schemaId, + BinaryRow row, + long logRecordTimestamp) { + if (timestampProvider != null) { + timestampProvider.setLogRecordTimestampMs(logRecordTimestamp); + } + return valueEncoder.createValue(schemaId, row); + } + private void initSchema(int schemaId) throws Exception { // todo, may need a cache, // but now, we get the schema from zk diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java index 0683f5e054..3a0a90045d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java @@ -21,6 +21,7 @@ import org.apache.fluss.compression.ArrowCompressionInfo; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.DeletionDisabledException; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.exception.KvStorageException; @@ -45,7 +46,9 @@ import org.apache.fluss.row.PaddingRow; import org.apache.fluss.row.arrow.ArrowWriterPool; import org.apache.fluss.row.arrow.ArrowWriterProvider; +import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.row.encode.ValueDecoder; +import org.apache.fluss.row.encode.ValueEncoder; import org.apache.fluss.rpc.protocol.MergeMode; import org.apache.fluss.server.kv.autoinc.AutoIncIDRange; import org.apache.fluss.server.kv.autoinc.AutoIncrementManager; @@ -78,7 +81,11 @@ import org.apache.fluss.utils.BytesUtils; import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.IOUtils; +import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.clock.SystemClock; +import org.rocksdb.AbstractCompactionFilter; +import org.rocksdb.AbstractCompactionFilterFactory; import org.rocksdb.RateLimiter; import org.rocksdb.ReadOptions; import org.rocksdb.RocksIterator; @@ -92,13 +99,16 @@ import java.io.File; import java.io.IOException; +import java.time.Duration; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.Executor; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock; import static org.apache.fluss.utils.concurrent.LockUtils.inWriteLock; @@ -125,6 +135,9 @@ public final class KvTablet { private final ReadWriteLock kvLock = new ReentrantReadWriteLock(); private final LogFormat logFormat; private final KvFormat kvFormat; + private final int kvFormatVersion; + private final ValueEncoder valueEncoder; + @Nullable private final RowTtlTimestampProvider rowTtlTimestampProvider; // defines how to merge rows on the same primary key private final RowMerger rowMerger; // Pre-created DefaultRowMerger for OVERWRITE mode (undo recovery scenarios) @@ -138,6 +151,8 @@ public final class KvTablet { // the changelog image mode for this tablet private final ChangelogImage changelogImage; + private final boolean rowTtlEnabled; + // RocksDB statistics accessor for this tablet @Nullable private final RocksDBStatistics rocksDBStatistics; @@ -168,8 +183,11 @@ private KvTablet( ArrowCompressionInfo arrowCompressionInfo, SchemaGetter schemaGetter, ChangelogImage changelogImage, + int kvFormatVersion, @Nullable RocksDBStatistics rocksDBStatistics, - AutoIncrementManager autoIncrementManager) { + AutoIncrementManager autoIncrementManager, + @Nullable RowTtlTimestampProvider rowTtlTimestampProvider, + boolean rowTtlEnabled) { this.physicalPath = physicalPath; this.tableBucket = tableBucket; this.logTablet = logTablet; @@ -182,6 +200,10 @@ private KvTablet( this.arrowWriterProvider = new ArrowWriterPool(arrowBufferAllocator); this.memorySegmentPool = memorySegmentPool; this.kvFormat = kvFormat; + this.kvFormatVersion = kvFormatVersion; + this.rowTtlTimestampProvider = rowTtlTimestampProvider; + this.valueEncoder = + ValueEncoder.forKvFormatVersion(kvFormatVersion, rowTtlTimestampProvider); this.rowMerger = rowMerger; // Pre-create DefaultRowMerger for OVERWRITE mode to avoid creating new instances // on every putAsLeader call. Used for undo recovery scenarios. @@ -189,10 +211,12 @@ private KvTablet( this.arrowCompressionInfo = arrowCompressionInfo; this.schemaGetter = schemaGetter; this.changelogImage = changelogImage; + this.rowTtlEnabled = rowTtlEnabled; this.rocksDBStatistics = rocksDBStatistics; this.autoIncrementManager = autoIncrementManager; - // disable row count for WAL image mode. - this.rowCount = changelogImage == ChangelogImage.WAL ? ROW_COUNT_DISABLED : 0L; + // Disable row count when writes or native cleanup can bypass exact count maintenance. + this.rowCount = + changelogImage == ChangelogImage.WAL || rowTtlEnabled ? ROW_COUNT_DISABLED : 0L; } public static KvTablet create( @@ -212,7 +236,144 @@ public static KvTablet create( RateLimiter sharedRateLimiter, AutoIncrementManager autoIncrementManager) throws IOException { - RocksDBKv kv = buildRocksDBKv(serverConf, kvTabletDir, sharedRateLimiter); + return create( + tablePath, + tableBucket, + logTablet, + kvTabletDir, + serverConf, + serverMetricGroup, + arrowBufferAllocator, + memorySegmentPool, + kvFormat, + rowMerger, + arrowCompressionInfo, + schemaGetter, + changelogImage, + sharedRateLimiter, + autoIncrementManager, + SystemClock.getInstance(), + ConfigOptions.KV_FORMAT_VERSION_2, + tableConfigWithRowTTL(Optional.empty())); + } + + public static KvTablet create( + PhysicalTablePath tablePath, + TableBucket tableBucket, + LogTablet logTablet, + File kvTabletDir, + Configuration serverConf, + TabletServerMetricGroup serverMetricGroup, + BufferAllocator arrowBufferAllocator, + MemorySegmentPool memorySegmentPool, + KvFormat kvFormat, + RowMerger rowMerger, + ArrowCompressionInfo arrowCompressionInfo, + SchemaGetter schemaGetter, + ChangelogImage changelogImage, + RateLimiter sharedRateLimiter, + AutoIncrementManager autoIncrementManager, + int kvFormatVersion, + Optional rowTtl) + throws IOException { + return create( + tablePath, + tableBucket, + logTablet, + kvTabletDir, + serverConf, + serverMetricGroup, + arrowBufferAllocator, + memorySegmentPool, + kvFormat, + rowMerger, + arrowCompressionInfo, + schemaGetter, + changelogImage, + sharedRateLimiter, + autoIncrementManager, + SystemClock.getInstance(), + kvFormatVersion, + tableConfigWithRowTTL(rowTtl)); + } + + public static KvTablet create( + PhysicalTablePath tablePath, + TableBucket tableBucket, + LogTablet logTablet, + File kvTabletDir, + Configuration serverConf, + TabletServerMetricGroup serverMetricGroup, + BufferAllocator arrowBufferAllocator, + MemorySegmentPool memorySegmentPool, + KvFormat kvFormat, + RowMerger rowMerger, + ArrowCompressionInfo arrowCompressionInfo, + SchemaGetter schemaGetter, + ChangelogImage changelogImage, + RateLimiter sharedRateLimiter, + AutoIncrementManager autoIncrementManager, + Clock clock, + int kvFormatVersion, + Optional rowTtl) + throws IOException { + return create( + tablePath, + tableBucket, + logTablet, + kvTabletDir, + serverConf, + serverMetricGroup, + arrowBufferAllocator, + memorySegmentPool, + kvFormat, + rowMerger, + arrowCompressionInfo, + schemaGetter, + changelogImage, + sharedRateLimiter, + autoIncrementManager, + clock, + kvFormatVersion, + tableConfigWithRowTTL(rowTtl)); + } + + public static KvTablet create( + PhysicalTablePath tablePath, + TableBucket tableBucket, + LogTablet logTablet, + File kvTabletDir, + Configuration serverConf, + TabletServerMetricGroup serverMetricGroup, + BufferAllocator arrowBufferAllocator, + MemorySegmentPool memorySegmentPool, + KvFormat kvFormat, + RowMerger rowMerger, + ArrowCompressionInfo arrowCompressionInfo, + SchemaGetter schemaGetter, + ChangelogImage changelogImage, + RateLimiter sharedRateLimiter, + AutoIncrementManager autoIncrementManager, + Clock clock, + int kvFormatVersion, + TableConfig tableConfig) + throws IOException { + checkNotNull(tableConfig, "tableConfig must not be null."); + Optional rowTtl = tableConfig.getRowTTL(); + @Nullable + AbstractCompactionFilterFactory> + compactionFilterFactory = + rowTtl.isPresent() + ? RowTtlCompactionFilterFactory.create(rowTtl.get(), clock) + : null; + KvValueLayout kvValueLayout = KvValueLayout.forKvFormatVersion(kvFormatVersion); + @Nullable + RowTtlTimestampProvider rowTtlTimestampProvider = + kvValueLayout.hasValueTag() + ? RowTtlTimestampProvider.forWrite(tableConfig, schemaGetter, clock) + : null; + RocksDBKv kv = + buildRocksDBKv(serverConf, kvTabletDir, sharedRateLimiter, compactionFilterFactory); // Create RocksDB statistics accessor (will be registered to TableMetricGroup by Replica) // Pass ResourceGuard to ensure thread-safe access during concurrent close operations @@ -242,12 +403,29 @@ public static KvTablet create( arrowCompressionInfo, schemaGetter, changelogImage, + kvFormatVersion, rocksDBStatistics, - autoIncrementManager); + autoIncrementManager, + rowTtlTimestampProvider, + rowTtl.isPresent()); + } + + private static TableConfig tableConfigWithRowTTL(Optional rowTtl) { + checkNotNull(rowTtl, "rowTtl must not be null."); + Configuration configuration = new Configuration(); + if (rowTtl.isPresent()) { + configuration.set(ConfigOptions.TABLE_KV_ROW_TTL, rowTtl.get()); + } + return new TableConfig(configuration); } private static RocksDBKv buildRocksDBKv( - Configuration configuration, File kvDir, RateLimiter sharedRateLimiter) + Configuration configuration, + File kvDir, + RateLimiter sharedRateLimiter, + @Nullable + AbstractCompactionFilterFactory> + compactionFilterFactory) throws IOException { // Enable statistics to support RocksDB statistics collection RocksDBResourceContainer rocksDBResourceContainer = @@ -257,9 +435,16 @@ private static RocksDBKv buildRocksDBKv( kvDir, rocksDBResourceContainer, rocksDBResourceContainer.getColumnOptions()); + if (compactionFilterFactory != null) { + rocksDBKvBuilder.setCompactionFilterFactory(compactionFilterFactory); + } return rocksDBKvBuilder.build(); } + ValueEncoder getValueEncoder() { + return valueEncoder; + } + public TableBucket getTableBucket() { return tableBucket; } @@ -300,16 +485,24 @@ void setFlushedLogOffset(long flushedLogOffset) { } void setRowCount(long rowCount) { - this.rowCount = rowCount; + if (this.rowCount != ROW_COUNT_DISABLED) { + this.rowCount = rowCount; + } } // row_count is volatile, so it's safe to read without lock public long getRowCount() { if (rowCount == ROW_COUNT_DISABLED) { + if (rowTtlEnabled) { + throw new InvalidTableException( + String.format( + "Row count is disabled for this table '%s' because row TTL cleanup does not maintain exact row count.", + getTablePath())); + } throw new InvalidTableException( String.format( "Row count is disabled for this table '%s'. This usually happens when the table is" - + "created before v0.9 or the changelog image is set to WAL, " + + " created before v0.9 or the changelog image is set to WAL, " + "as maintaining row count in WAL mode is costly and not necessary for most use cases. " + "If you want to enable row count, please set changelog image to FULL.", getTablePath())); @@ -410,6 +603,9 @@ public LogAppendInfo putAsLeader( long logEndOffsetOfPrevBatch = logTablet.localLogEndOffset(); try { + if (rowTtlTimestampProvider != null) { + rowTtlTimestampProvider.prepareForWriteBatch(); + } processKvRecords( kvRecords, kvRecords.schemaId(), @@ -478,13 +674,14 @@ private void processKvRecords( // TODO: reuse the read context and decoder KvRecordBatch.ReadContext readContext = KvRecordReadContext.createReadContext(kvFormat, schemaGetter); - ValueDecoder valueDecoder = new ValueDecoder(schemaGetter, kvFormat); + ValueDecoder valueDecoder = new ValueDecoder(schemaGetter, kvFormat, kvFormatVersion); for (KvRecord kvRecord : kvRecords.records(readContext)) { byte[] keyBytes = BytesUtils.toArray(kvRecord.getKey()); KvPreWriteBuffer.Key key = KvPreWriteBuffer.Key.of(keyBytes); BinaryRow row = kvRecord.getRow(); - BinaryValue currentValue = row == null ? null : new BinaryValue(schemaIdOfNewData, row); + BinaryValue currentValue = + row == null ? null : valueEncoder.createValue(schemaIdOfNewData, row); if (currentValue == null) { logOffset = @@ -611,6 +808,7 @@ private long applyInsert( AutoIncrementUpdater autoIncrementUpdater) throws Exception { BinaryValue newValue = autoIncrementUpdater.updateAutoIncrementColumns(currentValue); + newValue = refreshValueTag(newValue); walBuilder.append(ChangeType.INSERT, latestSchemaRow.replaceRow(newValue.row)); kvPreWriteBuffer.insert(key, newValue.encodeValue(), logOffset); return logOffset + 1; @@ -624,6 +822,7 @@ private long applyUpdate( PaddingRow latestSchemaRow, long logOffset) throws Exception { + newValue = refreshValueTag(newValue); if (changelogImage == ChangelogImage.WAL) { walBuilder.append(ChangeType.UPDATE_AFTER, latestSchemaRow.replaceRow(newValue.row)); kvPreWriteBuffer.update(key, newValue.encodeValue(), logOffset); @@ -636,6 +835,12 @@ private long applyUpdate( } } + private BinaryValue refreshValueTag(BinaryValue value) { + return valueEncoder.hasValueTag() + ? valueEncoder.createValue(value.schemaId, value.row) + : value; + } + private WalBuilder createWalBuilder(int schemaId, RowType rowType) throws Exception { switch (logFormat) { case INDEXED: diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java new file mode 100644 index 0000000000..79f6c1052d --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlCompactionFilterFactory.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.annotation.VisibleForTesting; +import org.apache.fluss.row.encode.KvValueLayout; +import org.apache.fluss.server.utils.RowTtlUtils; +import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.clock.SystemClock; + +import org.rocksdb.FlinkCompactionFilter; +import org.rocksdb.RocksDB; + +import java.time.Duration; + +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Factory utilities for row TTL compaction filters. */ +public final class RowTtlCompactionFilterFactory { + + private static final long QUERY_TIME_AFTER_NUM_ENTRIES = 1000L; + + private RowTtlCompactionFilterFactory() {} + + /** Creates a configured native compaction filter factory for row TTL cleanup. */ + public static FlinkCompactionFilter.FlinkCompactionFilterFactory create(Duration ttl) { + return create(ttl, QUERY_TIME_AFTER_NUM_ENTRIES, SystemClock.getInstance()); + } + + /** Creates a configured native compaction filter factory for row TTL cleanup. */ + public static FlinkCompactionFilter.FlinkCompactionFilterFactory create( + Duration ttl, Clock clock) { + return create(ttl, QUERY_TIME_AFTER_NUM_ENTRIES, clock); + } + + @VisibleForTesting + static FlinkCompactionFilter.FlinkCompactionFilterFactory create( + Duration ttl, long queryTimeAfterNumEntries) { + return create(ttl, queryTimeAfterNumEntries, SystemClock.getInstance()); + } + + @VisibleForTesting + static FlinkCompactionFilter.FlinkCompactionFilterFactory create( + Duration ttl, long queryTimeAfterNumEntries, Clock clock) { + long ttlMillis = RowTtlUtils.validateAndCeilTtlDurationToMillis(ttl); + checkNotNull(clock, "clock must not be null."); + checkArgument( + queryTimeAfterNumEntries > 0, + "queryTimeAfterNumEntries must be greater than zero."); + + RocksDB.loadLibrary(); + FlinkCompactionFilter.FlinkCompactionFilterFactory factory = + new FlinkCompactionFilter.FlinkCompactionFilterFactory(clock::milliseconds); + factory.configure( + FlinkCompactionFilter.Config.createNotList( + FlinkCompactionFilter.StateType.Value, + KvValueLayout.forKvFormatVersion(KV_FORMAT_VERSION_3).valueTagOffset(), + ttlMillis, + queryTimeAfterNumEntries)); + return factory; + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlTimestampProvider.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlTimestampProvider.java new file mode 100644 index 0000000000..06b0ccfda0 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/RowTtlTimestampProvider.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaGetter; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.fluss.utils.clock.Clock; + +import java.util.List; +import java.util.Optional; +import java.util.function.ToLongFunction; + +import static org.apache.fluss.types.DataTypeChecks.getPrecision; +import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; + +/** Provides the timestamp used as the V3 KV value tag for row TTL. */ +final class RowTtlTimestampProvider implements ToLongFunction { + + static final long NEVER_EXPIRE_TIMESTAMP_MS = Long.MAX_VALUE / 2; + + private final TimestampExtractor timestampExtractor; + private long timestampMs; + + private RowTtlTimestampProvider(TimestampExtractor timestampExtractor) { + this.timestampExtractor = timestampExtractor; + } + + static RowTtlTimestampProvider forWrite( + TableConfig tableConfig, SchemaGetter schemaGetter, Clock clock) { + checkNotNull(clock, "clock must not be null."); + if (tableConfig.getRowTTLTimeColumn().isPresent()) { + return forEventTime(tableConfig, schemaGetter); + } + return new RowTtlTimestampProvider(new BatchClockTimestampExtractor(clock)); + } + + static RowTtlTimestampProvider forRecovery(TableConfig tableConfig, SchemaGetter schemaGetter) { + if (tableConfig.getRowTTLTimeColumn().isPresent()) { + return forEventTime(tableConfig, schemaGetter); + } + return new RowTtlTimestampProvider(new LogRecordTimestampExtractor()); + } + + private static RowTtlTimestampProvider forEventTime( + TableConfig tableConfig, SchemaGetter schemaGetter) { + Optional timeColumnId = tableConfig.getRowTTLTimeColumnId(); + checkState( + timeColumnId.isPresent(), "Event-time row TTL requires internal time column id."); + return new RowTtlTimestampProvider( + EventTimeTimestampExtractor.create(schemaGetter, timeColumnId.get())); + } + + void prepareForWriteBatch() { + timestampExtractor.prepareForWriteBatch(this); + } + + void setLogRecordTimestampMs(long timestampMs) { + timestampExtractor.setLogRecordTimestampMs(this, timestampMs); + } + + @Override + public long applyAsLong(BinaryRow row) { + return timestampExtractor.extract(row, timestampMs); + } + + private interface TimestampExtractor { + long extract(BinaryRow row, long timestampMs); + + default void prepareForWriteBatch(RowTtlTimestampProvider provider) {} + + default void setLogRecordTimestampMs(RowTtlTimestampProvider provider, long timestampMs) {} + } + + private static final class BatchClockTimestampExtractor implements TimestampExtractor { + private final Clock clock; + + private BatchClockTimestampExtractor(Clock clock) { + this.clock = clock; + } + + @Override + public long extract(BinaryRow row, long timestampMs) { + return timestampMs; + } + + @Override + public void prepareForWriteBatch(RowTtlTimestampProvider provider) { + provider.timestampMs = clock.milliseconds(); + } + } + + private static final class LogRecordTimestampExtractor implements TimestampExtractor { + @Override + public long extract(BinaryRow row, long timestampMs) { + return timestampMs; + } + + @Override + public void setLogRecordTimestampMs(RowTtlTimestampProvider provider, long timestampMs) { + provider.timestampMs = timestampMs; + } + } + + private static final class EventTimeTimestampExtractor implements TimestampExtractor { + private final int fieldIndex; + private final DataType timeColumnType; + + private EventTimeTimestampExtractor(int fieldIndex, DataType timeColumnType) { + this.fieldIndex = fieldIndex; + this.timeColumnType = timeColumnType; + } + + private static EventTimeTimestampExtractor create( + SchemaGetter schemaGetter, int timeColumnId) { + Schema schema = schemaGetter.getLatestSchemaInfo().getSchema(); + List columns = schema.getColumns(); + for (int i = 0; i < columns.size(); i++) { + Schema.Column column = columns.get(i); + if (column.getColumnId() == timeColumnId) { + return new EventTimeTimestampExtractor(i, column.getDataType()); + } + } + throw new IllegalStateException( + String.format( + "Cannot find row TTL time column id %s in latest schema.", + timeColumnId)); + } + + @Override + public long extract(BinaryRow row, long timestampMs) { + if (row.isNullAt(fieldIndex)) { + return NEVER_EXPIRE_TIMESTAMP_MS; + } + DataTypeRoot typeRoot = timeColumnType.getTypeRoot(); + if (typeRoot == DataTypeRoot.BIGINT) { + return row.getLong(fieldIndex); + } + if (typeRoot == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + return row.getTimestampLtz(fieldIndex, getPrecision(timeColumnType)) + .getEpochMillisecond(); + } + throw new IllegalStateException( + String.format("Unsupported row TTL time column type: %s.", timeColumnType)); + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/autoinc/PerSchemaAutoIncrementUpdater.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/autoinc/PerSchemaAutoIncrementUpdater.java index a07698a65c..b4ab3653cc 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/autoinc/PerSchemaAutoIncrementUpdater.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/autoinc/PerSchemaAutoIncrementUpdater.java @@ -95,7 +95,7 @@ public BinaryValue updateAutoIncrementColumns(BinaryValue rowValue) { rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(rowValue.row)); } } - return new BinaryValue(schemaId, rowEncoder.finishRow()); + return rowValue.withRow(schemaId, rowEncoder.finishRow()); } @Override diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java index a7ce4bac9c..b85dda1141 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java @@ -123,7 +123,7 @@ public BinaryValue updateRow(@Nullable BinaryValue oldValue, BinaryValue partial } } } - return new BinaryValue(targetSchemaId, rowEncoder.finishRow()); + return partialValue.withRow(targetSchemaId, rowEncoder.finishRow()); } /** @@ -156,7 +156,7 @@ public BinaryValue updateRow(@Nullable BinaryValue oldValue, BinaryValue partial } } } - return new BinaryValue(targetSchemaId, rowEncoder.finishRow()); + return value.withRow(targetSchemaId, rowEncoder.finishRow()); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBKvBuilder.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBKvBuilder.java index 92e86fb964..52e57ebce7 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBKvBuilder.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBKvBuilder.java @@ -24,6 +24,8 @@ import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.IOUtils; +import org.rocksdb.AbstractCompactionFilter; +import org.rocksdb.AbstractCompactionFilterFactory; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.ColumnFamilyOptions; import org.rocksdb.NativeLibraryLoader; @@ -76,6 +78,13 @@ public RocksDBKvBuilder( this.instanceRocksDBPath = getInstanceRocksDBPath(instanceBasePath); } + @SuppressWarnings("rawtypes") + public RocksDBKvBuilder setCompactionFilterFactory( + AbstractCompactionFilterFactory> factory) { + columnFamilyOptions.setCompactionFilterFactory((AbstractCompactionFilterFactory) factory); + return this; + } + public RocksDBKv build() throws KvBuildingException { ColumnFamilyHandle defaultColumnFamilyHandle = null; RocksDB db = null; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMerger.java index f84769eb2e..5936128049 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMerger.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMerger.java @@ -105,7 +105,7 @@ public BinaryValue merge(@Nullable BinaryValue oldValue, BinaryValue newValue) { oldValue.row, newValue.row, oldContext, newContext, targetContext, encoder); BinaryRow mergedRow = encoder.finishRow(); - return new BinaryValue(targetSchemaId, mergedRow); + return newValue.withRow(targetSchemaId, mergedRow); } @Override @@ -294,7 +294,7 @@ public BinaryValue merge(@Nullable BinaryValue oldValue, BinaryValue newValue) { encoder); BinaryRow mergedRow = encoder.finishRow(); - return new BinaryValue(targetSchemaId, mergedRow); + return newValue.withRow(targetSchemaId, mergedRow); } @Override @@ -318,7 +318,7 @@ public BinaryValue delete(BinaryValue oldValue) { AggregateFieldsProcessor.encodePartialDeleteWithSameSchema( oldValue.row, context, targetPosBitSet, pkPosBitSet, encoder); BinaryRow deletedRow = encoder.finishRow(); - return new BinaryValue(targetSchemaId, deletedRow); + return oldValue.withRow(targetSchemaId, deletedRow); } // Schema evolution path: oldValue uses different schema @@ -344,7 +344,7 @@ public BinaryValue delete(BinaryValue oldValue) { targetPkPosBitSet, encoder); BinaryRow deletedRow = encoder.finishRow(); - return new BinaryValue(targetSchemaId, deletedRow); + return oldValue.withRow(targetSchemaId, deletedRow); } @Override diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index e11dd69c14..ea5bbdc950 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -788,7 +788,8 @@ private Optional initKvTablet() { checkNotNull(kvTablet, "kv tablet should not be null."); restoreStartOffset = completedSnapshot.getLogOffset(); - rowCount = completedSnapshot.getRowCount(); + rowCount = + supportsExactRowCount(tableConfig) ? completedSnapshot.getRowCount() : null; // currently, we only support one auto-increment column. autoIncIDRange = completedSnapshot.getFirstAutoIncIDRange(); } else { @@ -809,8 +810,7 @@ private Optional initKvTablet() { tableConfig, arrowCompressionInfo); - // we don't support rowCount - rowCount = tableConfig.getChangelogImage() == ChangelogImage.WAL ? null : 0L; + rowCount = supportsExactRowCount(tableConfig) ? 0L : null; // TODO: it is possible that this is a recovered kv tablet without kv snapshot but // with changelogs, in this case, the kv tablet should also have the // autoIncIDRange, we may need to get it from the changelog in the future. @@ -919,6 +919,7 @@ private void recoverKvTablet( recoverContext, tableConfig.getKvFormat(), tableConfig.getLogFormat(), + tableConfig, schemaGetter, remoteLogFetcher); kvRecoverHelper.recover(); @@ -1528,6 +1529,11 @@ public long getRowCount() { }); } + private static boolean supportsExactRowCount(TableConfig tableConfig) { + return tableConfig.getChangelogImage() != ChangelogImage.WAL + && !tableConfig.getRowTTL().isPresent(); + } + public long getOffset(RemoteLogManager remoteLogManager, ListOffsetsParam listOffsetsParam) throws IOException { return inReadLock( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 29adf341d9..fbdece28d8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -793,7 +793,7 @@ public void lookups( TableBucket tb = entry.getKey(); try { Replica replica = getReplicaOrException(tb); - validateClientVersionForPkTable(apiVersion, replica.getTableInfo()); + validateClientVersionForPkTable(ApiKeys.LOOKUP, apiVersion, replica.getTableInfo()); tableMetrics = replica.tableMetrics(); tableMetrics.totalLookupRequests().inc(); lookupResultForBucketMap.put( @@ -919,7 +919,8 @@ public void prefixLookups( List> resultForBucket = new ArrayList<>(); try { Replica replica = getReplicaOrException(tb); - validateClientVersionForPkTable(apiVersion, replica.getTableInfo()); + validateClientVersionForPkTable( + ApiKeys.PREFIX_LOOKUP, apiVersion, replica.getTableInfo()); tableMetrics = replica.tableMetrics(); tableMetrics.totalPrefixLookupRequests().inc(); for (byte[] prefixKey : entry.getValue()) { @@ -1344,7 +1345,7 @@ private Map putToLocalKv( try { LOG.trace("Put records to local kv tablet for table bucket {}", tb); Replica replica = getReplicaOrException(tb); - validateClientVersionForPkTable(apiVersion, replica.getTableInfo()); + validateClientVersionForPkTable(ApiKeys.PUT_KV, apiVersion, replica.getTableInfo()); tableMetrics = replica.tableMetrics(); tableMetrics.totalPutKvRequests().inc(); LogAppendInfo appendInfo = @@ -1402,6 +1403,7 @@ public void getTableStats( public void limitScan( TableBucket tableBucket, int limit, + short apiVersion, Consumer responseCallback) { LimitScanResultForBucket limitScanResultForBucket; TableMetricGroup tableMetrics = null; @@ -1410,6 +1412,8 @@ public void limitScan( tableMetrics = replica.tableMetrics(); tableMetrics.totalLimitScanRequests().inc(); if (replica.isKvTable()) { + validateClientVersionForPkTable( + ApiKeys.LIMIT_SCAN, apiVersion, replica.getTableInfo()); limitScanResultForBucket = new LimitScanResultForBucket(tableBucket, replica.limitKvScan(limit)); } else { @@ -2205,29 +2209,64 @@ public void shutdown() throws InterruptedException { checkpointHighWatermarks(); } - private void validateClientVersionForPkTable(int apiVersion, TableInfo tableInfo) { - if (apiVersion > 0) { - return; - } + /** + * Returns whether this API version is new enough to access any primary-key table without + * reading table-specific compatibility metadata. + */ + public static boolean canSkipClientVersionValidation(ApiKeys apiKey, int apiVersion) { + return apiVersion >= Math.max(1, requiredApiVersionForRowTTL(apiKey)); + } - // in the old version + /** Validates whether a client API version can access this primary-key table. */ + public static void validateClientVersionForPkTable( + ApiKeys apiKey, int apiVersion, TableInfo tableInfo) { TableConfig tableConfig = tableInfo.getTableConfig(); - // is with datalake format - if (tableConfig.getDataLakeFormat().isPresent()) { - Optional kvFormatVersion = tableConfig.getKvFormatVersion(); - if (kvFormatVersion.isPresent() - && kvFormatVersion.get() == KV_FORMAT_VERSION_2 - && !tableInfo.isDefaultBucketKey()) { + Optional kvFormatVersion = tableConfig.getKvFormatVersion(); + if (apiVersion < 1 + && tableConfig.getDataLakeFormat().isPresent() + && kvFormatVersion.isPresent() + && kvFormatVersion.get() == KV_FORMAT_VERSION_2 + && !tableInfo.isDefaultBucketKey()) { + throw new UnsupportedVersionException( + String.format( + "Client API version %d is not supported for table '%s'. " + + "This table uses new key encoding strategy (kv format version %d). " + + "Please upgrade your Fluss client to a newer version.", + apiVersion, tableInfo.getTablePath(), kvFormatVersion.get())); + } + + if (tableConfig.getRowTTL().isPresent()) { + int requiredVersion = requiredApiVersionForRowTTL(apiKey); + if (apiVersion < requiredVersion) { throw new UnsupportedVersionException( String.format( - "Client API version %d is not supported for table '%s'. " - + "This table uses new key encoding strategy (kv format version %d). " + "Client API version %d is not supported for table '%s' and API '%s'. " + + "This table has row TTL enabled. " + + "It requires API version %d or higher. " + "Please upgrade your Fluss client to a newer version.", - apiVersion, tableInfo.getTablePath(), kvFormatVersion.get())); + apiVersion, tableInfo.getTablePath(), apiKey, requiredVersion)); } } } + private static int requiredApiVersionForRowTTL(ApiKeys apiKey) { + switch (apiKey) { + case PUT_KV: + case LOOKUP: + case PREFIX_LOOKUP: + return 2; + case GET_LATEST_KV_SNAPSHOTS: + case GET_KV_SNAPSHOT_METADATA: + case ACQUIRE_KV_SNAPSHOT_LEASE: + case LIMIT_SCAN: + case SCAN_KV: + return 1; + default: + throw new IllegalArgumentException( + "Unsupported primary-key table client version check for API " + apiKey); + } + } + /** The result of reading log. */ public static final class LogReadResult { private final FetchLogResultForBucket fetchLogResultForBucket; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletServer.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletServer.java index c7c5fdd783..8270367bbd 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletServer.java @@ -244,7 +244,12 @@ protected void startServices() throws Exception { this.kvManager = KvManager.create( - conf, zkClient, logManager, tabletServerMetricGroup, localDiskManager); + conf, + zkClient, + logManager, + tabletServerMetricGroup, + localDiskManager, + clock); kvManager.startup(); // Register kvManager to dynamicConfigManager for dynamic reconfiguration diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index 892bcbc348..6b5c5a5555 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -27,6 +27,7 @@ import org.apache.fluss.exception.UnknownTableOrBucketException; import org.apache.fluss.fs.FileSystem; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.DefaultValueRecordBatch; import org.apache.fluss.record.KvRecordBatch; @@ -75,6 +76,7 @@ import org.apache.fluss.rpc.messages.UpdateMetadataRequest; import org.apache.fluss.rpc.messages.UpdateMetadataResponse; import org.apache.fluss.rpc.protocol.ApiError; +import org.apache.fluss.rpc.protocol.ApiKeys; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.rpc.protocol.MergeMode; import org.apache.fluss.security.acl.OperationType; @@ -340,6 +342,7 @@ public CompletableFuture limitScan(LimitScanRequest request) request.hasPartitionId() ? request.getPartitionId() : null, request.getBucketId()), request.getLimit(), + currentSession().getApiVersion(), value -> response.complete(makeLimitScanResponse(value))); return response; } @@ -536,9 +539,9 @@ public CompletableFuture scanKv(ScanKvRequest request) { bucketReq.getBucketId()); Long limit = bucketReq.hasLimit() ? bucketReq.getLimit() : null; - OpenScanResult openResult = - scannerManager.createScanner( - replicaManager.getReplicaOrException(tableBucket), limit); + Replica replica = replicaManager.getReplicaOrException(tableBucket); + validateClientVersionForPkTable(ApiKeys.SCAN_KV, replica.getTableInfo()); + OpenScanResult openResult = scannerManager.createScanner(replica, limit); isNewScan = true; initialLogOffset = openResult.getLogOffset(); @@ -610,8 +613,9 @@ public CompletableFuture scanKv(ScanKvRequest request) { // Catch a leadership flip ahead of the eventual closeScannersForBucket callback so // the client can redirect rather than consume a stale snapshot. + Replica replica = replicaManager.getReplicaOrException(context.getTableBucket()); + validateClientVersionForPkTable(ApiKeys.SCAN_KV, replica.getTableInfo()); if (!request.hasBucketScanReq()) { - Replica replica = replicaManager.getReplicaOrException(context.getTableBucket()); if (!replica.isLeader()) { throw new NotLeaderOrFollowerException( String.format( @@ -699,6 +703,19 @@ public void authorizeTable(OperationType operationType, long tableId) { } } + @Override + protected TableInfo getTableInfo(long tableId) { + TablePath tablePath = metadataCache.getTablePath(tableId).orElse(null); + if (tablePath == null) { + throw new UnknownTableOrBucketException( + String.format( + "This server %s does not know this table ID %s. This may happen when the table " + + "metadata cache in the server is not updated yet.", + serviceName, tableId)); + } + return metadataManager.getTable(tablePath); + } + private void authorizeAnyTable(OperationType operationType, List tablePaths) { if (authorizer != null) { if (tablePaths.isEmpty()) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/RowTtlUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/RowTtlUtils.java new file mode 100644 index 0000000000..375496012b --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/RowTtlUtils.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.utils; + +import org.apache.fluss.config.ConfigOptions; + +import java.time.Duration; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Utility methods for row-level TTL. */ +public final class RowTtlUtils { + + private RowTtlUtils() {} + + /** Validates a row TTL duration and returns its millisecond value rounded up. */ + public static long validateAndCeilTtlDurationToMillis(Duration ttl) { + checkNotNull(ttl, "ttl must not be null."); + checkArgument( + ttl.compareTo(Duration.ZERO) > 0, + "'%s' must be positive.", + ConfigOptions.TABLE_KV_ROW_TTL.key()); + + try { + return Math.addExact( + Math.multiplyExact(ttl.getSeconds(), 1000L), + (ttl.getNano() + 999_999L) / 1_000_000L); + } catch (ArithmeticException e) { + throw new IllegalArgumentException( + String.format( + "'%s' exceeds the maximum supported row TTL.", + ConfigOptions.TABLE_KV_ROW_TTL.key()), + e); + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 4be0a05347..0b84ee8837 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -34,6 +34,7 @@ import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.MergeEngineType; +import org.apache.fluss.metadata.RowTtlChangelogMode; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; @@ -45,6 +46,7 @@ import javax.annotation.Nullable; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -54,6 +56,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; +import static org.apache.fluss.config.ConfigOptions.MAX_KV_FORMAT_VERSION; import static org.apache.fluss.config.FlussConfigUtils.TABLE_OPTIONS; import static org.apache.fluss.config.FlussConfigUtils.isAlterableTableOption; import static org.apache.fluss.config.FlussConfigUtils.isTableStorageConfig; @@ -95,6 +99,9 @@ public static void validateTableDescriptor( // check properties should only contain table.* options, // and this cluster know it, and value is valid for (String key : tableConf.keySet()) { + if (TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY.equals(key)) { + continue; + } if (!TABLE_OPTIONS.containsKey(key)) { if (isTableStorageConfig(key)) { @@ -124,6 +131,9 @@ public static void validateTableDescriptor( checkMergeEngine(tableConf, hasPrimaryKey, schema); checkDeleteBehavior(tableConf, hasPrimaryKey); checkTieredLog(tableConf); + checkRowTTL(tableConf, schema, hasPrimaryKey); + checkKvFormatVersion( + tableConf, tableConf.getOptional(ConfigOptions.TABLE_KV_ROW_TTL).isPresent()); checkPartition(tableConf, tableDescriptor.getPartitionKeys(), schema.getRowType()); checkSystemColumns(schema.getRowType()); validateStatisticsConfig(tableDescriptor); @@ -242,6 +252,155 @@ private static void checkSystemColumns(RowType schema) { } } + private static void checkRowTTL(Configuration tableConf, Schema schema, boolean hasPrimaryKey) { + Optional rowTTL = tableConf.getOptional(ConfigOptions.TABLE_KV_ROW_TTL); + Optional timeColumn = + tableConf.getOptional(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN); + Optional timeColumnId = + Optional.ofNullable( + tableConf.toMap().get(TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY)); + + if (timeColumn.isPresent() && !rowTTL.isPresent()) { + throw new InvalidConfigException( + String.format( + "'%s' requires '%s' to be set.", + ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key(), + ConfigOptions.TABLE_KV_ROW_TTL.key())); + } + + if (timeColumnId.isPresent() && !timeColumn.isPresent()) { + throw new InvalidConfigException( + String.format( + "'%s' requires '%s' to be set.", + TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY, + ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key())); + } + + if (!rowTTL.isPresent()) { + return; + } + + if (!hasPrimaryKey) { + throw new InvalidTableException( + String.format( + "'%s' is only supported for primary key tables.", + ConfigOptions.TABLE_KV_ROW_TTL.key())); + } + + validateRowTTLDuration(rowTTL.get()); + + if (timeColumn.isPresent()) { + Schema.Column column = getRowTTLTimeColumn(schema, timeColumn.get()); + validateRowTTLTimeColumnType(column.getDataType()); + if (timeColumnId.isPresent()) { + validateRowTTLTimeColumnId(timeColumnId.get(), column.getColumnId()); + } + } + + RowTtlChangelogMode changelogMode = + tableConf.get(ConfigOptions.TABLE_KV_ROW_TTL_CHANGELOG_MODE); + if (changelogMode != RowTtlChangelogMode.NONE) { + throw new InvalidConfigException( + String.format( + "'%s' only supports '%s' in this version.", + ConfigOptions.TABLE_KV_ROW_TTL_CHANGELOG_MODE.key(), + RowTtlChangelogMode.NONE)); + } + } + + private static Schema.Column getRowTTLTimeColumn(Schema schema, String timeColumn) { + for (Schema.Column column : schema.getColumns()) { + if (column.getName().equals(timeColumn)) { + return column; + } + } + throw new InvalidConfigException( + String.format( + "'%s' refers to unknown column '%s'.", + ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key(), timeColumn)); + } + + private static void validateRowTTLTimeColumnType(DataType dataType) { + if (dataType.is(DataTypeRoot.BIGINT) + || dataType.is(DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) { + return; + } + throw new InvalidConfigException( + String.format( + "'%s' only supports BIGINT or TIMESTAMP_LTZ columns, but was %s.", + ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key(), dataType)); + } + + private static void validateRowTTLTimeColumnId(String configuredColumnId, int actualColumnId) { + try { + int parsedColumnId = Integer.parseInt(configuredColumnId); + if (parsedColumnId == actualColumnId) { + return; + } + } catch (NumberFormatException e) { + throw new InvalidConfigException( + String.format( + "Invalid value for '%s': %s.", + TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY, configuredColumnId)); + } + throw new InvalidConfigException( + String.format( + "'%s' must match the column id of '%s'.", + TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY, + ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key())); + } + + private static void validateRowTTLDuration(Duration ttl) { + try { + RowTtlUtils.validateAndCeilTtlDurationToMillis(ttl); + } catch (IllegalArgumentException e) { + throw new InvalidConfigException( + String.format( + "Invalid value for '%s': %s", + ConfigOptions.TABLE_KV_ROW_TTL.key(), e.getMessage())); + } + } + + private static void checkKvFormatVersion(Configuration tableConf, boolean rowTtlEnabled) { + Optional kvFormatVersion = + tableConf.getOptional(ConfigOptions.TABLE_KV_FORMAT_VERSION); + if (!kvFormatVersion.isPresent()) { + if (rowTtlEnabled) { + throw new InvalidConfigException( + String.format( + "'%s' must be set to %d when '%s' is set.", + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + KV_FORMAT_VERSION_3, + ConfigOptions.TABLE_KV_ROW_TTL.key())); + } + return; + } + + int version = kvFormatVersion.get(); + if (version > MAX_KV_FORMAT_VERSION) { + throw new InvalidConfigException( + String.format( + "Unsupported kv format version %d. The maximum supported version is %d.", + version, MAX_KV_FORMAT_VERSION)); + } + if (rowTtlEnabled && version < KV_FORMAT_VERSION_3) { + throw new InvalidConfigException( + String.format( + "'%s' must be at least %d when '%s' is set.", + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + KV_FORMAT_VERSION_3, + ConfigOptions.TABLE_KV_ROW_TTL.key())); + } + if (!rowTtlEnabled && version >= KV_FORMAT_VERSION_3) { + throw new InvalidConfigException( + String.format( + "'%s' version %d requires '%s' in this version.", + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + version, + ConfigOptions.TABLE_KV_ROW_TTL.key())); + } + } + private static void checkDistribution(TableDescriptor tableDescriptor, int maxBucketNum) { if (!tableDescriptor.getTableDistribution().isPresent()) { throw new InvalidTableException("Table distribution is required."); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerITCase.java index 0428d4e102..e18b97861e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerITCase.java @@ -23,6 +23,7 @@ import org.apache.fluss.config.AutoPartitionTimeUnit; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.DatabaseAlreadyExistException; import org.apache.fluss.exception.DatabaseNotEmptyException; import org.apache.fluss.exception.DatabaseNotExistException; @@ -95,6 +96,7 @@ import static org.apache.fluss.config.ConfigOptions.CURRENT_KV_FORMAT_VERSION; import static org.apache.fluss.config.ConfigOptions.DEFAULT_LISTENER_NAME; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateDatabaseRequest; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateTableRequest; @@ -679,6 +681,72 @@ void testMetadataCompatibility(boolean isCoordinatorServer) throws Exception { FLUSS_CLUSTER_EXTENSION.getTabletServerNodes()); } + @Test + void testCreateRowTTLTableDefaultsToKvFormatVersion3() throws Exception { + AdminReadOnlyGateway gateway = getAdminOnlyGateway(true); + AdminGateway adminGateway = getAdminGateway(); + + String db1 = "db_row_ttl"; + String tb1 = "tb_row_ttl"; + TablePath tablePath = TablePath.of(db1, tb1); + adminGateway.createDatabase(newCreateDatabaseRequest(db1, false)).get(); + + TableDescriptor tableDescriptor = + newPkTable() + .withProperties( + Collections.singletonMap( + ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h")); + adminGateway.createTable(newCreateTableRequest(tablePath, tableDescriptor, false)).get(); + + GetTableInfoResponse response = + gateway.getTableInfo(newGetTableInfoRequest(tablePath)).get(); + TableDescriptor gottenTable = TableDescriptor.fromJsonBytes(response.getTableJson()); + + assertThat(gottenTable.getProperties()) + .containsEntry( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(KV_FORMAT_VERSION_3)); + } + + @Test + void testCreateEventTimeRowTTLTableStoresTimeColumnId() throws Exception { + AdminReadOnlyGateway gateway = getAdminOnlyGateway(true); + AdminGateway adminGateway = getAdminGateway(); + + String db1 = "db_event_time_row_ttl"; + String tb1 = "tb_event_time_row_ttl"; + TablePath tablePath = TablePath.of(db1, tb1); + adminGateway.createDatabase(newCreateDatabaseRequest(db1, false)).get(); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("event_time", DataTypes.BIGINT()) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build(); + TableDescriptor tableDescriptor = + TableDescriptor.builder() + .schema(schema) + .distributedBy(3, "id") + .property(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h") + .property(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key(), "event_time") + .build(); + adminGateway.createTable(newCreateTableRequest(tablePath, tableDescriptor, false)).get(); + + GetTableInfoResponse response = + gateway.getTableInfo(newGetTableInfoRequest(tablePath)).get(); + TableDescriptor gottenTable = TableDescriptor.fromJsonBytes(response.getTableJson()); + + assertThat(gottenTable.getProperties()) + .containsEntry( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(KV_FORMAT_VERSION_3)) + .containsEntry( + TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY, + String.valueOf(schema.getColumn("event_time").getColumnId())); + } + @Test void testSchemaEvolution() throws Exception { AdminReadOnlyGateway gateway = getAdminOnlyGateway(true); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvRecoverHelperTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvRecoverHelperTest.java new file mode 100644 index 0000000000..d2d10966d7 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvRecoverHelperTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.record.BinaryValue; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.row.encode.ValueEncoder; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link KvRecoverHelper}. */ +class KvRecoverHelperTest { + + @Test + void testRecoveredVersion3ValueUsesLogRecordTimestamp() { + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + TestingSchemaGetter schemaGetter = + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, DEFAULT_SCHEMA_ID)); + RowTtlTimestampProvider timestampProvider = + RowTtlTimestampProvider.forRecovery(rowTtlProcessTimeConfig(), schemaGetter); + ValueEncoder writeEncoder = + ValueEncoder.forKvFormatVersion(KV_FORMAT_VERSION_3, timestampProvider); + + BinaryValue recoveredValue = + KvRecoverHelper.createRecoveredValue( + writeEncoder, timestampProvider, DEFAULT_SCHEMA_ID, row, 200L); + + assertThat(recoveredValue.getValueTag()).isEqualTo(200L); + assertThat(recoveredValue.row.getInt(0)).isEqualTo(1); + assertThat(recoveredValue.row.getString(1).toString()).isEqualTo("a"); + } + + @Test + void testRecoveredEventTimeVersion3ValueUsesTimeColumn() { + Schema schema = eventTimeSchema(); + TableConfig tableConfig = rowTtlEventTimeConfig(schema); + TestingSchemaGetter schemaGetter = + new TestingSchemaGetter(new SchemaInfo(schema, DEFAULT_SCHEMA_ID)); + RowTtlTimestampProvider timestampProvider = + RowTtlTimestampProvider.forRecovery(tableConfig, schemaGetter); + ValueEncoder writeEncoder = + ValueEncoder.forKvFormatVersion(KV_FORMAT_VERSION_3, timestampProvider); + BinaryRow row = compactedRow(schema.getRowType(), new Object[] {1, 1234L, "a"}); + + BinaryValue recoveredValue = + KvRecoverHelper.createRecoveredValue( + writeEncoder, timestampProvider, DEFAULT_SCHEMA_ID, row, 200L); + + assertThat(recoveredValue.getValueTag()).isEqualTo(1234L); + assertThat(recoveredValue.row.getInt(0)).isEqualTo(1); + assertThat(recoveredValue.row.getLong(1)).isEqualTo(1234L); + assertThat(recoveredValue.row.getString(2).toString()).isEqualTo("a"); + } + + private static TableConfig rowTtlProcessTimeConfig() { + Configuration configuration = new Configuration(); + configuration.set(ConfigOptions.TABLE_KV_ROW_TTL, java.time.Duration.ofHours(1)); + configuration.set(ConfigOptions.TABLE_KV_FORMAT_VERSION, KV_FORMAT_VERSION_3); + return new TableConfig(configuration); + } + + private static Schema eventTimeSchema() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("event_time", DataTypes.BIGINT()) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build(); + } + + private static TableConfig rowTtlEventTimeConfig(Schema schema) { + Configuration configuration = new Configuration(); + configuration.set(ConfigOptions.TABLE_KV_ROW_TTL, java.time.Duration.ofHours(1)); + configuration.set(ConfigOptions.TABLE_KV_FORMAT_VERSION, KV_FORMAT_VERSION_3); + configuration.setString(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN, "event_time"); + configuration.setString( + TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY, + String.valueOf(schema.getColumn("event_time").getColumnId())); + return new TableConfig(configuration); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java index 3425fa75cf..30966bf914 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java @@ -33,6 +33,7 @@ import org.apache.fluss.metadata.SchemaInfo; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.BinaryValue; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.FileLogProjection; import org.apache.fluss.record.KvRecord; @@ -49,6 +50,7 @@ import org.apache.fluss.record.TestingSchemaGetter; import org.apache.fluss.record.bytesview.MultiBytesView; import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.row.encode.ValueDecoder; import org.apache.fluss.row.encode.ValueEncoder; import org.apache.fluss.server.kv.autoinc.AutoIncrementManager; import org.apache.fluss.server.kv.autoinc.TestingSequenceGeneratorFactory; @@ -71,6 +73,8 @@ import org.apache.fluss.types.RowType; import org.apache.fluss.types.StringType; import org.apache.fluss.utils.CloseableIterator; +import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.clock.ManualClock; import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.FlussScheduler; @@ -84,6 +88,7 @@ import javax.annotation.Nullable; import java.io.File; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -146,14 +151,26 @@ void afterEach() { private void initLogTabletAndKvTablet(Schema schema, Map tableConfig) throws Exception { - initLogTabletAndKvTablet(TablePath.of("testDb", "t1"), schema, tableConfig); + initLogTabletAndKvTablet( + TablePath.of("testDb", "t1"), schema, tableConfig, SystemClock.getInstance()); + } + + private void initLogTabletAndKvTablet( + Schema schema, Map tableConfig, Clock clock) throws Exception { + initLogTabletAndKvTablet(TablePath.of("testDb", "t1"), schema, tableConfig, clock); } private void initLogTabletAndKvTablet( TablePath tablePath, Schema schema, Map tableConfig) throws Exception { + initLogTabletAndKvTablet(tablePath, schema, tableConfig, SystemClock.getInstance()); + } + + private void initLogTabletAndKvTablet( + TablePath tablePath, Schema schema, Map tableConfig, Clock clock) + throws Exception { PhysicalTablePath physicalTablePath = PhysicalTablePath.of(tablePath); schemaGetter = new TestingSchemaGetter(new SchemaInfo(schema, schemaId)); - logTablet = createLogTablet(tempLogDir, 0L, physicalTablePath); + logTablet = createLogTablet(tempLogDir, 0L, physicalTablePath, clock); TableBucket tableBucket = logTablet.getTableBucket(); kvTablet = createKvTablet( @@ -162,11 +179,18 @@ private void initLogTabletAndKvTablet( logTablet, tmpKvDir, schemaGetter, - tableConfig); + tableConfig, + clock); } private LogTablet createLogTablet(File tempLogDir, long tableId, PhysicalTablePath tablePath) throws Exception { + return createLogTablet(tempLogDir, tableId, tablePath, SystemClock.getInstance()); + } + + private LogTablet createLogTablet( + File tempLogDir, long tableId, PhysicalTablePath tablePath, Clock clock) + throws Exception { File logTabletDir = LogTestUtils.makeRandomLogTabletDir( tempLogDir, tablePath.getDatabaseName(), tableId, tablePath.getTableName()); @@ -181,7 +205,7 @@ private LogTablet createLogTablet(File tempLogDir, long tableId, PhysicalTablePa LogFormat.ARROW, 1, true, - SystemClock.getInstance(), + clock, true); } @@ -193,6 +217,25 @@ private KvTablet createKvTablet( SchemaGetter schemaGetter, Map tableConfig) throws Exception { + return createKvTablet( + tablePath, + tableBucket, + logTablet, + tmpKvDir, + schemaGetter, + tableConfig, + SystemClock.getInstance()); + } + + private KvTablet createKvTablet( + PhysicalTablePath tablePath, + TableBucket tableBucket, + LogTablet logTablet, + File tmpKvDir, + SchemaGetter schemaGetter, + Map tableConfig, + Clock clock) + throws Exception { TableConfig tableConf = new TableConfig(Configuration.fromMap(tableConfig)); RowMerger rowMerger = RowMerger.create(tableConf, KvFormat.COMPACTED, schemaGetter); AutoIncrementManager autoIncrementManager = @@ -217,7 +260,182 @@ private KvTablet createKvTablet( schemaGetter, tableConf.getChangelogImage(), KvManager.getDefaultRateLimiter(), - autoIncrementManager); + autoIncrementManager, + clock, + tableConf.getKvFormatVersion().orElse(ConfigOptions.KV_FORMAT_VERSION_2), + tableConf); + } + + @Test + void testRowTtlValueTimestampUsesTabletClock() throws Exception { + long writeTimestampMs = 123456789L; + ManualClock clock = new ManualClock(writeTimestampMs); + Map tableConfig = new HashMap<>(); + tableConfig.put(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h"); + tableConfig.put( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(ConfigOptions.KV_FORMAT_VERSION_3)); + initLogTabletAndKvTablet(DATA1_SCHEMA_PK, tableConfig, clock); + + KvRecord record = kvRecordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); + kvTablet.putAsLeader( + kvRecordBatchFactory.ofRecords(Collections.singletonList(record)), null); + kvTablet.flush(Long.MAX_VALUE, NOPErrorHandler.INSTANCE); + + byte[] value = kvTablet.multiGet(Collections.singletonList("k1".getBytes())).get(0); + BinaryValue decoded = + new ValueDecoder( + schemaGetter, KvFormat.COMPACTED, ConfigOptions.KV_FORMAT_VERSION_3) + .decodeValue(value); + + assertThat(decoded.getValueTag()).isEqualTo(writeTimestampMs); + assertThat(decoded.row.getInt(0)).isEqualTo(1); + assertThat(decoded.row.getString(1).toString()).isEqualTo("a"); + } + + @Test + void testRowTtlEventTimeValueTimestampUsesTimeColumn() throws Exception { + Schema eventTimeSchema = eventTimeSchema(); + Map tableConfig = rowTtlEventTimeConfig(eventTimeSchema); + initLogTabletAndKvTablet(eventTimeSchema, tableConfig, new ManualClock(123456789L)); + + KvRecordTestUtils.KvRecordFactory eventTimeRecordFactory = + KvRecordTestUtils.KvRecordFactory.of(eventTimeSchema.getRowType()); + KvRecord record = + eventTimeRecordFactory.ofRecord( + "k1".getBytes(), new Object[] {1, 1234L, "event-time-row"}); + kvTablet.putAsLeader( + kvRecordBatchFactory.ofRecords(Collections.singletonList(record)), null); + kvTablet.flush(Long.MAX_VALUE, NOPErrorHandler.INSTANCE); + + BinaryValue decoded = decodeVersion3Value("k1"); + + assertThat(decoded.getValueTag()).isEqualTo(1234L); + assertThat(decoded.row.getInt(0)).isEqualTo(1); + assertThat(decoded.row.getLong(1)).isEqualTo(1234L); + assertThat(decoded.row.getString(2).toString()).isEqualTo("event-time-row"); + } + + @Test + void testRowTtlNullEventTimeValueNeverExpires() throws Exception { + Schema eventTimeSchema = eventTimeSchema(); + Map tableConfig = rowTtlEventTimeConfig(eventTimeSchema); + initLogTabletAndKvTablet(eventTimeSchema, tableConfig, new ManualClock(123456789L)); + + KvRecordTestUtils.KvRecordFactory eventTimeRecordFactory = + KvRecordTestUtils.KvRecordFactory.of(eventTimeSchema.getRowType()); + KvRecord record = + eventTimeRecordFactory.ofRecord( + "k1".getBytes(), new Object[] {1, null, "null-event-time-row"}); + kvTablet.putAsLeader( + kvRecordBatchFactory.ofRecords(Collections.singletonList(record)), null); + kvTablet.flush(Long.MAX_VALUE, NOPErrorHandler.INSTANCE); + + BinaryValue decoded = decodeVersion3Value("k1"); + + assertThat(decoded.getValueTag()) + .isEqualTo(RowTtlTimestampProvider.NEVER_EXPIRE_TIMESTAMP_MS); + assertThat(decoded.row.getInt(0)).isEqualTo(1); + assertThat(decoded.row.isNullAt(1)).isTrue(); + assertThat(decoded.row.getString(2).toString()).isEqualTo("null-event-time-row"); + } + + @Test + void testPartialUpdateFromNullEventTimeExpiresWholeRowAfterCompaction() throws Exception { + long eventTimestampMs = 1000L; + ManualClock clock = new ManualClock(eventTimestampMs); + Schema eventTimeSchema = eventTimeSchema(); + initLogTabletAndKvTablet(eventTimeSchema, rowTtlEventTimeConfig(eventTimeSchema), clock); + + String key = "partial-event-time"; + KvRecordTestUtils.KvRecordFactory eventTimeRecordFactory = + KvRecordTestUtils.KvRecordFactory.of(eventTimeSchema.getRowType()); + kvTablet.putAsLeader( + kvRecordBatchFactory.ofRecords( + eventTimeRecordFactory.ofRecord( + key.getBytes(), new Object[] {1, null, "retained-name"})), + null); + // Persist the never-expiring version before overwriting it with a partial update. + kvTablet.flush(Long.MAX_VALUE, NOPErrorHandler.INSTANCE); + + kvTablet.putAsLeader( + kvRecordBatchFactory.ofRecords( + eventTimeRecordFactory.ofRecord( + key.getBytes(), + new Object[] {1, eventTimestampMs, "ignored-name"})), + new int[] {0, 1}); + kvTablet.flush(Long.MAX_VALUE, NOPErrorHandler.INSTANCE); + + BinaryValue updatedValue = decodeVersion3Value(key); + assertThat(updatedValue.getValueTag()).isEqualTo(eventTimestampMs); + assertThat(updatedValue.row.getLong(1)).isEqualTo(eventTimestampMs); + assertThat(updatedValue.row.getString(2).toString()).isEqualTo("retained-name"); + + clock.advanceTime(Duration.ofHours(2L)); + kvTablet.getRocksDBKv().getDb().compactRange(); + + assertThat(kvTablet.multiGet(Collections.singletonList(key.getBytes())).get(0)).isNull(); + } + + @Test + void testRowTTLCompactionRemovesExpiredRowsFromLookupAndScan() throws Exception { + long writeTimestampMs = 1000L; + ManualClock clock = new ManualClock(writeTimestampMs); + Map tableConfig = new HashMap<>(); + tableConfig.put(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h"); + tableConfig.put( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(ConfigOptions.KV_FORMAT_VERSION_3)); + initLogTabletAndKvTablet(DATA1_SCHEMA_PK, tableConfig, clock); + + String expiredKey = "mm-expired-target"; + kvTablet.putAsLeader( + kvRecordBatchFactory.ofRecords( + kvRecordFactory.ofRecord( + expiredKey.getBytes(), new Object[] {0, "expired-target"})), + null); + kvTablet.flush(Long.MAX_VALUE, NOPErrorHandler.INSTANCE); + + clock.advanceTime(Duration.ofHours(2L)); + List freshRows = new ArrayList<>(); + int freshPrefixRows = 1001; + for (int i = 0; i < freshPrefixRows; i++) { + String key = String.format("aa-fresh-%04d", i); + freshRows.add( + kvRecordFactory.ofRecord( + key.getBytes(), new Object[] {i + 1, "fresh-prefix-" + i})); + } + String freshKey = "zz-fresh"; + freshRows.add(kvRecordFactory.ofRecord(freshKey.getBytes(), new Object[] {2000, "fresh"})); + kvTablet.putAsLeader(kvRecordBatchFactory.ofRecords(freshRows), null); + kvTablet.flush(Long.MAX_VALUE, NOPErrorHandler.INSTANCE); + + kvTablet.getRocksDBKv().getDb().compactRange(); + + List lookupValues = + kvTablet.multiGet(Arrays.asList(expiredKey.getBytes(), freshKey.getBytes())); + assertThat(lookupValues.get(0)).isNull(); + assertThat(lookupValues.get(1)).isNotNull(); + + ValueDecoder valueDecoder = + new ValueDecoder( + schemaGetter, KvFormat.COMPACTED, ConfigOptions.KV_FORMAT_VERSION_3); + OpenScanResult result = kvTablet.openScan("scanner-row-ttl", -1L, 0L); + ScannerContext context = result.getContext(); + assertThat(context).isNotNull(); + List scannedNames = new ArrayList<>(); + while (context.isValid()) { + BinaryValue value = valueDecoder.decodeValue(context.currentValue()); + scannedNames.add(value.row.getString(1).toString()); + context.advance(); + } + context.checkIteratorStatus(); + context.close(); + + assertThat(scannedNames) + .hasSize(freshPrefixRows + 1) + .doesNotContain("expired-target") + .contains("fresh"); } @Test @@ -1759,6 +1977,34 @@ void testRowCountBasic() throws Exception { kvTablet.close(); } + private BinaryValue decodeVersion3Value(String key) throws Exception { + byte[] value = kvTablet.multiGet(Collections.singletonList(key.getBytes())).get(0); + return new ValueDecoder(schemaGetter, KvFormat.COMPACTED, ConfigOptions.KV_FORMAT_VERSION_3) + .decodeValue(value); + } + + private static Schema eventTimeSchema() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("event_time", DataTypes.BIGINT()) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build(); + } + + private static Map rowTtlEventTimeConfig(Schema schema) { + Map tableConfig = new HashMap<>(); + tableConfig.put(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h"); + tableConfig.put( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(ConfigOptions.KV_FORMAT_VERSION_3)); + tableConfig.put(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key(), "event_time"); + tableConfig.put( + TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY, + String.valueOf(schema.getColumn("event_time").getColumnId())); + return tableConfig; + } + @Test void testRowCountWithUpsert() throws Exception { initLogTabletAndKvTablet(DATA1_SCHEMA_PK, new HashMap<>()); @@ -1815,6 +2061,29 @@ void testRowCountDisabledForWalChangelog() throws Exception { kvTablet.close(); } + @Test + void testRowCountDisabledForRowTTL() throws Exception { + Map tableConfig = new HashMap<>(); + tableConfig.put(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h"); + tableConfig.put( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(ConfigOptions.KV_FORMAT_VERSION_3)); + initLogTabletAndKvTablet(DATA1_SCHEMA_PK, tableConfig); + + KvRecordBatch batch = + kvRecordBatchFactory.ofRecords( + kvRecordFactory.ofRecord("key1", new Object[] {1, "val1"})); + kvTablet.putAsLeader(batch, null); + kvTablet.flush(Long.MAX_VALUE, NOPErrorHandler.INSTANCE); + + assertThatThrownBy(() -> kvTablet.getRowCount()) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("row TTL"); + assertThat(kvTablet.getTabletState().getRowCount()).isNull(); + + kvTablet.close(); + } + @Test void testRowCountWithMixedOperations() throws Exception { initLogTabletAndKvTablet(DATA1_SCHEMA_PK, new HashMap<>()); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java new file mode 100644 index 0000000000..8db5bbb7ec --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlCompactionFilterTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.rocksdb.RocksDBHandle; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.row.encode.ValueEncoder; +import org.apache.fluss.utils.clock.ManualClock; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.FlinkCompactionFilter; +import org.rocksdb.FlushOptions; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Duration; + +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests row TTL cleanup with native Flink compaction filter. */ +class RowTtlCompactionFilterTest { + + @TempDir private Path tempDir; + + @Test + void testFlinkCompactionFilterReadsTimestampFromVersion3Value() throws Exception { + FlinkCompactionFilter.FlinkCompactionFilterFactory filterFactory = + RowTtlCompactionFilterFactory.create( + Duration.ofHours(1L), 1L, new ManualClock(123456789L)); + byte[] expiredKey = "expired-key".getBytes(StandardCharsets.UTF_8); + byte[] freshKey = "fresh-key".getBytes(StandardCharsets.UTF_8); + BinaryRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"}); + long now = 123456789L; + + try (DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); + ColumnFamilyOptions cfOptions = + new ColumnFamilyOptions().setCompactionFilterFactory(filterFactory); + RocksDBHandle handle = new RocksDBHandle(tempDir.toFile(), dbOptions, cfOptions); + FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + handle.openDB(); + handle.getDb() + .put( + expiredKey, + ValueEncoder.encodeValueWithTag( + DEFAULT_SCHEMA_ID, now - Duration.ofHours(2L).toMillis(), row)); + handle.getDb() + .put(freshKey, ValueEncoder.encodeValueWithTag(DEFAULT_SCHEMA_ID, now, row)); + handle.getDb().flush(flushOptions); + + handle.getDb().compactRange(); + + assertThat(handle.getDb().get(expiredKey)).isNull(); + assertThat(handle.getDb().get(freshKey)).isNotNull(); + } + } + + @Test + void testCreateRejectsInvalidTtlDuration() { + assertThatThrownBy(() -> RowTtlCompactionFilterFactory.create(Duration.ZERO, 1L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(ConfigOptions.TABLE_KV_ROW_TTL.key()); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlTimestampProviderTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlTimestampProviderTest.java new file mode 100644 index 0000000000..6047dc06d7 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/RowTtlTimestampProviderTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.clock.ManualClock; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link RowTtlTimestampProvider}. */ +class RowTtlTimestampProviderTest { + + private static final short SCHEMA_ID = 0; + + @Test + void testProcessTimeWriteProviderUsesBatchClock() { + ManualClock clock = new ManualClock(1234L); + RowTtlTimestampProvider provider = + RowTtlTimestampProvider.forWrite( + processTimeTableConfig(), schemaGetter(schema()), clock); + + provider.prepareForWriteBatch(); + + assertThat(provider.applyAsLong(row(1, 100L))).isEqualTo(1234L); + + clock.advanceTime(Duration.ofMillis(100)); + assertThat(provider.applyAsLong(row(2, 200L))).isEqualTo(1234L); + } + + @Test + void testProcessTimeRecoveryProviderUsesLogRecordTimestamp() { + RowTtlTimestampProvider provider = + RowTtlTimestampProvider.forRecovery( + processTimeTableConfig(), schemaGetter(schema())); + + provider.setLogRecordTimestampMs(5678L); + + assertThat(provider.applyAsLong(row(1, 100L))).isEqualTo(5678L); + } + + @Test + void testBigintEventTimeProviderReadsEpochMillis() { + RowTtlTimestampProvider provider = + RowTtlTimestampProvider.forWrite( + eventTimeTableConfig(schema().getColumn("event_time").getColumnId()), + schemaGetter(schema()), + new ManualClock(0L)); + + assertThat(provider.applyAsLong(row(1, 1234L))).isEqualTo(1234L); + } + + @Test + void testTimestampLtzEventTimeProviderReadsEpochMillis() { + Schema timestampLtzSchema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("event_time", DataTypes.TIMESTAMP_LTZ(3)) + .primaryKey("id") + .build(); + RowTtlTimestampProvider provider = + RowTtlTimestampProvider.forWrite( + eventTimeTableConfig( + timestampLtzSchema.getColumn("event_time").getColumnId()), + schemaGetter(timestampLtzSchema), + new ManualClock(0L)); + + BinaryRow row = + compactedRow( + timestampLtzSchema.getRowType(), + new Object[] {1, TimestampLtz.fromEpochMillis(5678L)}); + + assertThat(provider.applyAsLong(row)).isEqualTo(5678L); + } + + @Test + void testNullEventTimeNeverExpires() { + RowTtlTimestampProvider provider = + RowTtlTimestampProvider.forWrite( + eventTimeTableConfig(schema().getColumn("event_time").getColumnId()), + schemaGetter(schema()), + new ManualClock(0L)); + + assertThat(provider.applyAsLong(row(1, null))) + .isEqualTo(RowTtlTimestampProvider.NEVER_EXPIRE_TIMESTAMP_MS); + } + + private static TableConfig processTimeTableConfig() { + Configuration configuration = new Configuration(); + configuration.set(ConfigOptions.TABLE_KV_ROW_TTL, java.time.Duration.ofHours(1)); + return new TableConfig(configuration); + } + + private static TableConfig eventTimeTableConfig(int timeColumnId) { + Configuration configuration = new Configuration(); + configuration.set(ConfigOptions.TABLE_KV_ROW_TTL, java.time.Duration.ofHours(1)); + configuration.setString(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN, "event_time"); + configuration.setString( + TableConfig.KV_ROW_TTL_TIME_COLUMN_ID_KEY, String.valueOf(timeColumnId)); + return new TableConfig(configuration); + } + + private static TestingSchemaGetter schemaGetter(Schema schema) { + return new TestingSchemaGetter(new SchemaInfo(schema, SCHEMA_ID)); + } + + private static Schema schema() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("event_time", DataTypes.BIGINT()) + .primaryKey("id") + .build(); + } + + private static BinaryRow row(int id, Long eventTime) { + return compactedRow(schema().getRowType(), new Object[] {id, eventTime}); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerRowTtlClientVersionTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerRowTtlClientVersionTest.java new file mode 100644 index 0000000000..58d090e04a --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerRowTtlClientVersionTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.exception.UnsupportedVersionException; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.TestData; +import org.apache.fluss.rpc.protocol.ApiKeys; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests row TTL client version checks in {@link ReplicaManager}. */ +class ReplicaManagerRowTtlClientVersionTest { + + @Test + void testRowTTLClientVersionRequirements() { + TableInfo tableInfo = rowTTLTableInfo(); + + assertRequiredVersion( + tableInfo, + 1, + ApiKeys.GET_LATEST_KV_SNAPSHOTS, + ApiKeys.GET_KV_SNAPSHOT_METADATA, + ApiKeys.ACQUIRE_KV_SNAPSHOT_LEASE, + ApiKeys.LIMIT_SCAN, + ApiKeys.SCAN_KV); + assertRequiredVersion(tableInfo, 2, ApiKeys.PUT_KV, ApiKeys.LOOKUP, ApiKeys.PREFIX_LOOKUP); + } + + private static void assertRequiredVersion( + TableInfo tableInfo, int requiredVersion, ApiKeys... apiKeys) { + for (ApiKeys apiKey : apiKeys) { + assertThat(ReplicaManager.canSkipClientVersionValidation(apiKey, requiredVersion)) + .isTrue(); + assertThat(ReplicaManager.canSkipClientVersionValidation(apiKey, requiredVersion - 1)) + .isFalse(); + assertThatThrownBy( + () -> + ReplicaManager.validateClientVersionForPkTable( + apiKey, requiredVersion - 1, tableInfo)) + .isInstanceOf(UnsupportedVersionException.class) + .hasMessageContaining("row TTL") + .hasMessageContaining("API '" + apiKey + "'") + .hasMessageContaining("requires API version " + requiredVersion); + + ReplicaManager.validateClientVersionForPkTable(apiKey, requiredVersion, tableInfo); + } + } + + private static TableInfo rowTTLTableInfo() { + Map properties = + new HashMap<>(TestData.DATA1_TABLE_DESCRIPTOR_PK.getProperties()); + properties.put(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h"); + properties.put( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), String.valueOf(KV_FORMAT_VERSION_3)); + TableDescriptor descriptor = TestData.DATA1_TABLE_DESCRIPTOR_PK.withProperties(properties); + return TableInfo.of( + TablePath.of("db", "ttl_table"), + 1L, + 1, + descriptor, + TestData.DEFAULT_REMOTE_DATA_DIR, + 1L, + 1L); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java index 6dd9ce7fe5..6a839c3395 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java @@ -33,6 +33,7 @@ import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.SchemaGetter; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.ChangeType; @@ -116,6 +117,7 @@ import java.util.concurrent.TimeUnit; import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; import static org.apache.fluss.record.LogRecordReadContext.createArrowReadContext; import static org.apache.fluss.record.TestData.ANOTHER_DATA1; import static org.apache.fluss.record.TestData.DATA1; @@ -125,6 +127,7 @@ import static org.apache.fluss.record.TestData.DATA1_SCHEMA; import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_DESCRIPTOR; +import static org.apache.fluss.record.TestData.DATA1_TABLE_DESCRIPTOR_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; @@ -1312,7 +1315,7 @@ void testLimitScanPrimaryKeyTable() throws Exception { // first limit scan from an empty table. CompletableFuture future = new CompletableFuture<>(); - replicaManager.limitScan(tb, 1, future::complete); + replicaManager.limitScan(tb, 1, (short) 1, future::complete); assertThat(future.get().getValues()).isEqualTo(builder.build()); // first, send one batch kv. @@ -1330,17 +1333,109 @@ void testLimitScanPrimaryKeyTable() throws Exception { // second, limit scan from table with limit builder.append(DEFAULT_SCHEMA_ID, compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a1"})); future = new CompletableFuture<>(); - replicaManager.limitScan(tb, 1, future::complete); + replicaManager.limitScan(tb, 1, (short) 1, future::complete); assertThat(future.get().getValues()).isEqualTo(builder.build()); // third, limit scan from table with more limit future = new CompletableFuture<>(); - replicaManager.limitScan(tb, 3, future::complete); + replicaManager.limitScan(tb, 3, (short) 1, future::complete); // there is only 2 records in the table bucket after merged builder.append(DEFAULT_SCHEMA_ID, compactedRow(DATA1_ROW_TYPE, new Object[] {2, "b1"})); assertThat(future.get().getValues()).isEqualTo(builder.build()); } + @Test + void testLimitScanRequiresRowTTLAwareClient() throws Exception { + TablePath tablePath = TablePath.of("test_db", "ttl_limit_scan_table"); + long tableId = 150006L; + Map properties = new HashMap<>(DATA1_TABLE_DESCRIPTOR_PK.getProperties()); + properties.put(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h"); + properties.put( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), String.valueOf(KV_FORMAT_VERSION_3)); + TableDescriptor descriptor = DATA1_TABLE_DESCRIPTOR_PK.withProperties(properties); + + if (zkClient.tableExist(tablePath)) { + zkClient.deleteTable(tablePath); + } + zkClient.registerTable( + tablePath, + TableRegistration.newTable(tableId, DEFAULT_REMOTE_DATA_DIR, descriptor)); + zkClient.registerFirstSchema(tablePath, DATA1_SCHEMA_PK); + + TableBucket tableBucket = new TableBucket(tableId, 0); + makeKvTableAsLeader(tableId, tablePath, tableBucket.getBucket()); + + CompletableFuture future = new CompletableFuture<>(); + replicaManager.limitScan(tableBucket, 1, (short) 0, future::complete); + + LimitScanResultForBucket result = future.get(); + assertThat(result.failed()).isTrue(); + assertThat(result.getError().error()).isEqualTo(Errors.UNSUPPORTED_VERSION); + assertThat(result.getError().message()).contains("row TTL"); + } + + @Test + void testRowTTLTableRejectsV1RawKvClient() throws Exception { + TablePath tablePath = TablePath.of("test_db", "ttl_raw_kv_table"); + long tableId = 150007L; + Map properties = new HashMap<>(DATA1_TABLE_DESCRIPTOR_PK.getProperties()); + properties.put(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h"); + properties.put( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), String.valueOf(KV_FORMAT_VERSION_3)); + TableDescriptor descriptor = DATA1_TABLE_DESCRIPTOR_PK.withProperties(properties); + + if (zkClient.tableExist(tablePath)) { + zkClient.deleteTable(tablePath); + } + zkClient.registerTable( + tablePath, + TableRegistration.newTable(tableId, DEFAULT_REMOTE_DATA_DIR, descriptor)); + zkClient.registerFirstSchema(tablePath, DATA1_SCHEMA_PK); + + TableBucket tableBucket = new TableBucket(tableId, 0); + makeKvTableAsLeader(tableId, tablePath, tableBucket.getBucket()); + short legacyRawKvVersion = 1; + + CompletableFuture> putFuture = new CompletableFuture<>(); + replicaManager.putRecordsToKv( + 20000, + 1, + Collections.singletonMap(tableBucket, genKvRecordBatch(DATA_1_WITH_KEY_AND_VALUE)), + null, + MergeMode.DEFAULT, + legacyRawKvVersion, + putFuture::complete); + PutKvResultForBucket putResult = putFuture.get().get(0); + assertThat(putResult.failed()).isTrue(); + assertThat(putResult.getErrorCode()).isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + assertThat(putResult.getErrorMessage()).contains("row TTL"); + + CompactedKeyEncoder keyEncoder = new CompactedKeyEncoder(DATA1_ROW_TYPE, new int[] {0}); + byte[] keyBytes = keyEncoder.encodeKey(row(DATA_1_WITH_KEY_AND_VALUE.get(0).f0)); + + CompletableFuture> lookupFuture = + new CompletableFuture<>(); + replicaManager.lookups( + Collections.singletonMap(tableBucket, Collections.singletonList(keyBytes)), + legacyRawKvVersion, + lookupFuture::complete); + LookupResultForBucket lookupResult = lookupFuture.get().get(tableBucket); + assertThat(lookupResult.failed()).isTrue(); + assertThat(lookupResult.getErrorCode()).isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + assertThat(lookupResult.getErrorMessage()).contains("row TTL"); + + CompletableFuture> prefixFuture = + new CompletableFuture<>(); + replicaManager.prefixLookups( + Collections.singletonMap(tableBucket, Collections.singletonList(keyBytes)), + legacyRawKvVersion, + prefixFuture::complete); + PrefixLookupResultForBucket prefixResult = prefixFuture.get().get(tableBucket); + assertThat(prefixResult.failed()).isTrue(); + assertThat(prefixResult.getError().error()).isEqualTo(Errors.UNSUPPORTED_VERSION); + assertThat(prefixResult.getError().message()).contains("row TTL"); + } + @Test void testLimitScanLogTable() throws Exception { TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); @@ -1367,7 +1462,7 @@ void testLimitScanLogTable() throws Exception { // get limit 10 records from local. CompletableFuture limitFuture = new CompletableFuture<>(); - replicaManager.limitScan(tb, 10, limitFuture::complete); + replicaManager.limitScan(tb, 10, (short) 1, limitFuture::complete); SchemaGetter schemaGetter = serverMetadataCache.subscribeWithInitialSchema( DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, DATA1_SCHEMA); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java index bad818a9e9..16f1f6e459 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java @@ -221,7 +221,8 @@ public void setup(TestInfo testInfo) throws Exception { zkClient, logManager, TestingMetricGroups.TABLET_SERVER_METRICS, - localDiskManager); + localDiskManager, + manualClock); kvManager.startup(); serverMetadataCache = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/TableDescriptorValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/TableDescriptorValidationTest.java new file mode 100644 index 0000000000..ab161b9a6a --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/TableDescriptorValidationTest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.utils; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.exception.InvalidAlterTableException; +import org.apache.fluss.exception.InvalidConfigException; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.TestData; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; +import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_3; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link TableDescriptorValidation}. */ +class TableDescriptorValidationTest { + + @Test + void testCreatePkTableWithRowTTL() { + TableDescriptorValidation.validateTableDescriptor( + pkTableWithProperties( + ConfigOptions.TABLE_KV_ROW_TTL.key(), + "1 h", + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(KV_FORMAT_VERSION_3)), + 100, + null); + } + + @Test + void testCreateLogTableWithRowTTLFails() { + assertThatThrownBy( + () -> + TableDescriptorValidation.validateTableDescriptor( + logTableWithProperty( + ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h"), + 100, + null)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining(ConfigOptions.TABLE_KV_ROW_TTL.key()) + .hasMessageContaining("primary key"); + } + + @ParameterizedTest + @MethodSource("supportedRowTTLTimeColumnTypes") + void testCreateTableWithSupportedRowTTLTimeColumn(DataType timeColumnType) { + TableDescriptor descriptor = pkTableWithRowTTLTimeColumn("event_time", timeColumnType); + + TableDescriptorValidation.validateTableDescriptor(descriptor, 100, null); + } + + @Test + void testCreateTableWithMissingRowTTLTimeColumnFails() { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(TestData.DATA1_SCHEMA_PK) + .distributedBy(3) + .property(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h") + .property(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key(), "event_time") + .property( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(KV_FORMAT_VERSION_3)) + .build() + .withReplicationFactor(3); + + assertThatThrownBy( + () -> + TableDescriptorValidation.validateTableDescriptor( + descriptor, 100, null)) + .isInstanceOf(InvalidConfigException.class) + .hasMessageContaining(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key()); + } + + @ParameterizedTest + @MethodSource("unsupportedRowTTLTimeColumnTypes") + void testCreateTableWithUnsupportedRowTTLTimeColumnFails(DataType timeColumnType) { + assertThatThrownBy( + () -> + TableDescriptorValidation.validateTableDescriptor( + pkTableWithRowTTLTimeColumn("event_time", timeColumnType), + 100, + null)) + .isInstanceOf(InvalidConfigException.class) + .hasMessageContaining(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key()) + .hasMessageContaining("BIGINT") + .hasMessageContaining("TIMESTAMP_LTZ"); + } + + @Test + void testCreateTableWithLargeRowTTL() { + TableDescriptor descriptor = + pkTableWithProperties( + ConfigOptions.TABLE_KV_ROW_TTL.key(), + "3000000000 s", + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(KV_FORMAT_VERSION_3)); + + TableDescriptorValidation.validateTableDescriptor(descriptor, 100, null); + } + + @Test + void testCreateTableWithRowTTLMillisOverflowFails() { + TableDescriptor descriptor = + pkTableWithProperties( + ConfigOptions.TABLE_KV_ROW_TTL.key(), + "9223372036854776 s", + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(KV_FORMAT_VERSION_3)); + + assertThatThrownBy( + () -> + TableDescriptorValidation.validateTableDescriptor( + descriptor, 100, null)) + .isInstanceOf(InvalidConfigException.class) + .hasMessageContaining(ConfigOptions.TABLE_KV_ROW_TTL.key()) + .hasMessageContaining("exceeds"); + } + + @Test + void testCreateRowTTLTableWithKvFormatVersion2Fails() { + TableDescriptor descriptor = + pkTableWithProperties( + ConfigOptions.TABLE_KV_ROW_TTL.key(), + "1 h", + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(KV_FORMAT_VERSION_2)); + + assertThatThrownBy( + () -> + TableDescriptorValidation.validateTableDescriptor( + descriptor, 100, null)) + .isInstanceOf(InvalidConfigException.class) + .hasMessageContaining(ConfigOptions.TABLE_KV_FORMAT_VERSION.key()) + .hasMessageContaining(ConfigOptions.TABLE_KV_ROW_TTL.key()); + } + + @Test + void testCreateNonRowTTLTableWithKvFormatVersion3Fails() { + TableDescriptor descriptor = + pkTableWithProperty( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(KV_FORMAT_VERSION_3)); + + assertThatThrownBy( + () -> + TableDescriptorValidation.validateTableDescriptor( + descriptor, 100, null)) + .isInstanceOf(InvalidConfigException.class) + .hasMessageContaining(ConfigOptions.TABLE_KV_FORMAT_VERSION.key()) + .hasMessageContaining(ConfigOptions.TABLE_KV_ROW_TTL.key()); + } + + @Test + void testAlterRowTTLFails() { + TableInfo currentTable = + TableInfo.of( + TablePath.of("db", "t"), + 1L, + 1, + pkTableWithProperty(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h"), + "file://remote", + 1L, + 1L); + + assertThatThrownBy( + () -> + TableDescriptorValidation.validateAlterTableProperties( + currentTable, + Collections.singleton( + ConfigOptions.TABLE_KV_ROW_TTL.key()))) + .isInstanceOf(InvalidAlterTableException.class) + .hasMessageContaining(ConfigOptions.TABLE_KV_ROW_TTL.key()); + } + + private static Stream supportedRowTTLTimeColumnTypes() { + return Stream.of(Arguments.of(DataTypes.BIGINT()), Arguments.of(DataTypes.TIMESTAMP_LTZ())); + } + + private static Stream unsupportedRowTTLTimeColumnTypes() { + return Stream.of(Arguments.of(DataTypes.STRING()), Arguments.of(DataTypes.TIMESTAMP())); + } + + private TableDescriptor pkTableWithProperty(String key, String value) { + return TableDescriptor.builder() + .schema(TestData.DATA1_SCHEMA_PK) + .distributedBy(3) + .property(key, value) + .build() + .withReplicationFactor(3); + } + + private TableDescriptor pkTableWithProperties( + String key1, String value1, String key2, String value2) { + Map properties = new HashMap<>(); + properties.put(key1, value1); + properties.put(key2, value2); + return TableDescriptor.builder() + .schema(TestData.DATA1_SCHEMA_PK) + .distributedBy(3) + .properties(properties) + .build() + .withReplicationFactor(3); + } + + private TableDescriptor pkTableWithRowTTLTimeColumn( + String timeColumn, DataType timeColumnType) { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column(timeColumn, timeColumnType) + .primaryKey("id") + .build(); + return TableDescriptor.builder() + .schema(schema) + .distributedBy(3) + .property(ConfigOptions.TABLE_KV_ROW_TTL.key(), "1 h") + .property(ConfigOptions.TABLE_KV_ROW_TTL_TIME_COLUMN.key(), timeColumn) + .property( + ConfigOptions.TABLE_KV_FORMAT_VERSION.key(), + String.valueOf(KV_FORMAT_VERSION_3)) + .build() + .withReplicationFactor(3); + } + + private TableDescriptor logTableWithProperty(String key, String value) { + return TableDescriptor.builder() + .schema(TestData.DATA1_SCHEMA) + .distributedBy(3) + .property(key, value) + .build() + .withReplicationFactor(3); + } +} diff --git a/website/docs/engine-flink/ddl.md b/website/docs/engine-flink/ddl.md index 06f586f7ac..8a365755d0 100644 --- a/website/docs/engine-flink/ddl.md +++ b/website/docs/engine-flink/ddl.md @@ -259,7 +259,7 @@ ALTER TABLE MyTable ADD ( ### SET properties The SET statement allows users to configure one or more connector options including the [Storage Options](engine-flink/options.md#storage-options) for a specified table. If a particular option is already configured on the table, it will be overridden with the new value. -When using SET to modify [Storage Options](engine-flink/options.md#storage-options), the Fluss cluster will dynamically apply the changes to the table. This can be useful for modifying table behavior without needing to recreate the table. +When using SET to modify [Storage Options](engine-flink/options.md#storage-options), the Fluss cluster will apply the changes according to each option's runtime contract. This can be useful for modifying table behavior without needing to recreate the table. **Supported Options to modify** - All [Read Options](engine-flink/options.md#read-options), [Write Options](engine-flink/options.md#write-options), [Lookup Options](engine-flink/options.md#lookup-options) and [Other Options](engine-flink/options.md#other-options) except `bootstrap.servers`. @@ -283,6 +283,7 @@ ALTER TABLE my_table SET ('table.log.tiered.local-segments' = '5'); **Limits** - If lakehouse storage (`table.datalake.enabled`) is already enabled for a table, options with lakehouse format prefixes (e.g., `paimon.*`) cannot be modified again. +- `table.kv.row.ttl` cannot be modified with `ALTER TABLE ... SET` in this version. Configure row-level TTL when creating the primary-key table. ### RESET properties @@ -294,6 +295,8 @@ The following example illustrates reset the `table.datalake.enabled` option to i ALTER TABLE my_table RESET ('table.datalake.enabled'); ``` +`table.kv.row.ttl` cannot be reset to disable row-level TTL. To change or disable row-level TTL, create a new table with the desired TTL configuration and migrate the data. + ## Add Partition Fluss supports manually adding partitions to an existing partitioned table through the Fluss Catalog. If the specified partition diff --git a/website/docs/engine-flink/options.md b/website/docs/engine-flink/options.md index 329ea94f13..d31714e190 100644 --- a/website/docs/engine-flink/options.md +++ b/website/docs/engine-flink/options.md @@ -67,6 +67,9 @@ See more details about [ALTER TABLE ... SET](engine-flink/ddl.md#set-properties) | bucket.num | int | The bucket number of Fluss cluster. | The number of buckets of a Fluss table. | | bucket.key | String | (None) | Specific the distribution policy of the Fluss table. Data will be distributed to each bucket according to the hash value of bucket-key (It must be a subset of the primary keys excluding partition keys of the primary key table). If you specify multiple fields, delimiter is `,`. If the table has a primary key and a bucket key is not specified, the bucket key will be used as primary key(excluding the partition key). If the table has no primary key and the bucket key is not specified, the data will be distributed to each bucket randomly. | | table.log.ttl | Duration | 7 days | The time to live for log segments. The configuration controls the maximum time we will retain a log before we will delete old segments to free up space. If set to -1, the log will not be deleted. | +| table.kv.row.ttl | Duration | (None) | The best-effort row-level TTL for primary key tables. If this option is not set, row-level TTL is disabled. The value must be a positive duration. Expired rows may remain visible until RocksDB compaction removes them. TTL uses processing time by default, or event time when `table.kv.row.ttl.time-column` is configured. This option can only be configured when creating a primary key table; `ALTER TABLE ... SET` and `ALTER TABLE ... RESET` are not supported in this version. | +| table.kv.row.ttl.changelog-mode | Enum | none | The changelog mode for row-level TTL cleanup. Only `none` is supported in this version, which means TTL cleanup does not emit delete records to `$changelog` or `$binlog`. | +| table.kv.row.ttl.time-column | String | (None) | The event-time column for row-level TTL. If this option is not set, row TTL uses processing time. If set, `table.kv.row.ttl` must also be set, and the column must be a `BIGINT` epoch-millis column or a `TIMESTAMP_LTZ` column. Rows with null event-time values do not expire through row TTL. This option can only be configured when creating a primary key table; `ALTER TABLE ... SET` and `ALTER TABLE ... RESET` are not supported in this version. | | table.auto-partition.enabled | Boolean | false | Whether enable auto partition for the table. Disable by default. When auto partition is enabled, the partitions of the table will be created automatically. | | table.auto-partition.key | String | (None) | This configuration defines the time-based partition key to be used for auto-partitioning when a table is partitioned with multiple keys. Auto-partitioning utilizes a time-based partition key to handle partitions automatically, including creating new ones and removing outdated ones, by comparing the time value of the partition with the current system time. In the case of a table using multiple partition keys (such as a composite partitioning strategy), this feature determines which key should serve as the primary time dimension for making auto-partitioning decisions. And If the table has only one partition key, this config is not necessary. Otherwise, it must be specified. | | table.auto-partition.time-unit | ENUM | DAY | The time granularity for auto created partitions. The default value is `DAY`. Valid values are `HOUR`, `DAY`, `MONTH`, `QUARTER`, `YEAR`. If the value is `HOUR`, the partition format for auto created is yyyyMMddHH. If the value is `DAY`, the partition format for auto created is yyyyMMdd. If the value is `MONTH`, the partition format for auto created is yyyyMM. If the value is `QUARTER`, the partition format for auto created is yyyyQ. If the value is `YEAR`, the partition format for auto created is yyyy. | @@ -79,7 +82,7 @@ See more details about [ALTER TABLE ... SET](engine-flink/ddl.md#set-properties) | table.log.arrow.compression.type | Enum | ZSTD | The compression type of the log records if the log format is set to `ARROW`. The candidate compression type is `NONE`, `LZ4_FRAME`, `ZSTD`. The default value is `ZSTD`. | | table.log.arrow.compression.zstd.level | Integer | 3 | The compression level of the log records if the log format is set to `ARROW` and the compression type is set to `ZSTD`. The valid range is 1 to 22. The default value is 3. | | table.kv.format | Enum | COMPACTED | The format of the kv records in kv store. The default value is `COMPACTED`. The supported formats are `COMPACTED` and `INDEXED`. | -| table.kv.format-version | Integer | (None) | The version of the kv format. Automatically set by the coordinator during table creation if not configured by users.

**Note:** The datalake encoding and bucketing strategy mentioned below only takes effect when `datalake.format` is configured at cluster level.

**Version Behaviors:**

(1) **Version 1**: Tables created before `table.kv.format-version` was introduced are treated as version 1. Uses datalake's encoder (e.g., Paimon/Iceberg) for both primary key and bucket key encoding. This may not support prefix lookup properly because some datalake encoders (like Paimon) don't guarantee that encoded bucket key bytes are a prefix of encoded primary key bytes.

(2) **Version 2** (current): New tables use Fluss's default encoder for primary key encoding when bucket key differs from primary key, which ensures proper prefix lookup support. When bucket key equals primary key (default bucket key), it still uses datalake's encoder for optimization. Bucket key encoding always uses datalake's encoder to align with datalake bucket calculation. | +| table.kv.format-version | Integer | (None) | The version of the kv format. Automatically set by the coordinator during table creation if not configured by users.

**Note:** The datalake encoding and bucketing strategy mentioned below only takes effect when `datalake.format` is configured at cluster level.

**Version Behaviors:**

(1) **Version 1**: Tables created before `table.kv.format-version` was introduced are treated as version 1. Uses datalake's encoder (e.g., Paimon/Iceberg) for both primary key and bucket key encoding. This may not support prefix lookup properly because some datalake encoders (like Paimon) don't guarantee that encoded bucket key bytes are a prefix of encoded primary key bytes.

(2) **Version 2** (current): New non-TTL primary key tables use Fluss's default encoder for primary key encoding when bucket key differs from primary key, which ensures proper prefix lookup support. When bucket key equals primary key (default bucket key), it still uses datalake's encoder for optimization. Bucket key encoding always uses datalake's encoder to align with datalake bucket calculation.

(3) **Version 3**: Row TTL primary key tables encode a Fluss-owned timestamp before the row payload so RocksDB compaction filters can clean up expired rows. The timestamp is derived from processing time by default, or from `table.kv.row.ttl.time-column` for event-time TTL. The coordinator automatically uses version 3 when `table.kv.row.ttl` is set. Explicitly using version 3 without row TTL is not supported in this version. | | table.kv.standby-replica.enabled | Boolean | (None) | Whether to enable standby replicas for primary key tables. Standby replicas maintain recent KV snapshots for fast leader promotion. Automatically set to `true` by the coordinator during table creation for new PK tables. Tables created before this option was introduced are treated as disabled. Can be dynamically enabled via `ALTER TABLE SET ('table.kv.standby-replica.enabled' = 'true')`. | | table.log.tiered.local-segments | Integer | 2 | The number of log segments to retain in local for each table when log tiered storage is enabled. It must be greater that 0. The default is 2. | | table.datalake.enabled | Boolean | false | Whether enable lakehouse storage for the table. Disabled by default. When this option is set to ture and the datalake tiering service is up, the table will be tiered and compacted into datalake format stored on lakehouse storage. | diff --git a/website/docs/table-design/data-distribution/ttl.md b/website/docs/table-design/data-distribution/ttl.md index d356c1e334..bf862d2e5e 100644 --- a/website/docs/table-design/data-distribution/ttl.md +++ b/website/docs/table-design/data-distribution/ttl.md @@ -8,4 +8,46 @@ sidebar_position: 3 Fluss supports TTL for data by setting the TTL attribute for tables with `'table.log.ttl' = ''` (default is 7 days). Fluss can periodically and automatically check for and clean up expired data in the table. For log tables, this attribute indicates the expiration time of the log table data. -For primary key tables, this attribute indicates the expiration time of the changelog and does not represent the expiration time of the primary key table data. If you also want the data in the primary key table to expire automatically, please use [auto partitioning](partitioning.md#auto-partitioning). +For primary key tables, this attribute indicates the expiration time of the changelog and does not represent the expiration time of the primary key table data. + +## Row TTL for Primary Key Tables + +Primary key tables can configure row-level TTL with `'table.kv.row.ttl' = ''`. The option has no default value; if it is not configured, row-level TTL is disabled. + +```sql title="Flink SQL" +CREATE TABLE pk_table +( + id BIGINT, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '4', + 'table.kv.row.ttl' = '7 d' +); +``` + +Row TTL is best-effort cleanup. A row becomes eligible for cleanup after the configured duration, but expired rows may still be visible until RocksDB compaction removes them. Fluss stores the TTL timestamp in the primary-key table value and uses a compaction filter to remove expired rows during RocksDB compaction. + +TTL cleanup does not emit delete records. Downstream consumers of `$changelog` or `$binlog` will not receive delete changes when rows expire due to TTL. + +By default, row TTL uses processing time. To use event time, configure `table.kv.row.ttl.time-column` when creating the table: + +```sql title="Flink SQL" +CREATE TABLE pk_table_with_event_time +( + id BIGINT, + event_time BIGINT, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '4', + 'table.kv.row.ttl' = '7 d', + 'table.kv.row.ttl.time-column' = 'event_time' +); +``` + +The event-time column must be `BIGINT` epoch milliseconds or `TIMESTAMP_LTZ`. Rows with null event-time values do not expire through row TTL. + +Row TTL must be configured when creating the primary key table. Changing or disabling `table.kv.row.ttl`, or changing `table.kv.row.ttl.time-column`, with `ALTER TABLE ... SET` or `ALTER TABLE ... RESET` is not supported in this version. + +See [Flink Connector Options](/engine-flink/options.md#storage-options) for the complete row TTL option definitions.