From bdb65c5cb050858dc8d391ce35a741a33ac1429c Mon Sep 17 00:00:00 2001 From: Vladislav Pyatkov Date: Mon, 6 Jul 2026 22:06:38 +0300 Subject: [PATCH 1/6] IGNITE-28864 Add technical columns to Calcite engin SQL scan --- .../calcite/prepare/IgniteSqlValidator.java | 4 +- .../schema/CacheTableDescriptorImpl.java | 106 ++++++- .../calcite/schema/TechnicalColumns.java | 42 +++ .../integration/TechnicalColumnsScanTest.java | 263 ++++++++++++++++++ .../testsuites/IntegrationTestSuite.java | 2 + 5 files changed, 415 insertions(+), 2 deletions(-) create mode 100644 modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/TechnicalColumns.java create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java index 8d25cf1c249a0..b85e13a0e2c60 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java @@ -70,6 +70,7 @@ import org.apache.ignite.internal.processors.query.calcite.schema.CacheTableDescriptor; import org.apache.ignite.internal.processors.query.calcite.schema.IgniteCacheTable; import org.apache.ignite.internal.processors.query.calcite.schema.IgniteTable; +import org.apache.ignite.internal.processors.query.calcite.schema.TechnicalColumns; import org.apache.ignite.internal.processors.query.calcite.sql.IgniteSqlDecimalLiteral; import org.apache.ignite.internal.processors.query.calcite.type.IgniteTypeFactory; import org.apache.ignite.internal.processors.query.calcite.type.OtherType; @@ -554,7 +555,8 @@ private IgniteTypeFactory typeFactory() { /** */ private boolean isSystemFieldName(String alias) { return QueryUtils.KEY_FIELD_NAME.equalsIgnoreCase(alias) - || QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(alias); + || QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(alias) + || TechnicalColumns.isTechnicalFieldNameIgnoreCase(alias); } /** {@inheritDoc} */ diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java index 2cd164bc81570..17dedb0b56e2e 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java @@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.query.GridQueryProperty; import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor; import org.apache.ignite.internal.processors.query.QueryUtils; @@ -123,7 +124,7 @@ public CacheTableDescriptorImpl(GridCacheContextInfo cacheInfo, GridQueryT Set fields = this.typeDesc.fields().keySet(); - List descriptors = new ArrayList<>(fields.size() + 2); + List descriptors = new ArrayList<>(fields.size() + 4); // A _key/_val field is virtual in case there is an alias or a property(es) mapped to the _key/_val field. BitSet virtualFields = new BitSet(); @@ -177,6 +178,28 @@ else if (Objects.equals(field, typeDesc.valueFieldAlias())) { } } + virtualFields.set(descriptors.size()); + descriptors.add(new TechnicalDescriptor(TechnicalColumns.VER_FIELD_NAME, GridCacheVersion.class, descriptors.size()) { + @Override public Object value(ExecutionContext ectx, GridCacheContext cctx, CacheDataRow src) + throws IgniteCheckedException { + GridCacheVersion ver = src.version(); + + if (ver != null) + return ver; + + CacheDataRow row = cctx.offheap().read(cctx, src.key()); + + return row == null ? null : row.version(); + } + }); + + virtualFields.set(descriptors.size()); + descriptors.add(new TechnicalDescriptor(TechnicalColumns.SRC_FIELD_NAME, Integer.class, descriptors.size()) { + @Override public Object value(ExecutionContext ectx, GridCacheContext cctx, CacheDataRow src) { + return cctx.cacheId(); + } + }); + Map descriptorsMap = U.newHashMap(descriptors.size()); for (CacheColumnDescriptor descriptor : descriptors) descriptorsMap.put(descriptor.name(), descriptor); @@ -278,6 +301,9 @@ else if (affFields.isEmpty()) @Override public boolean isUpdateAllowed(RelOptTable tbl, int colIdx) { final CacheColumnDescriptor desc = descriptors[colIdx]; + if (isTechnicalColumn(desc.name())) + return false; + return !desc.key() && (desc.field() || QueryUtils.isSqlType(desc.storageType())); } @@ -289,6 +315,11 @@ else if (affFields.isEmpty()) return super.generationStrategy(tbl, colIdx); } + /** */ + private static boolean isTechnicalColumn(String fieldName) { + return TechnicalColumns.isTechnicalFieldName(fieldName); + } + /** {@inheritDoc} */ @Override public RexNode newColumnDefaultValue(RelOptTable tbl, int colIdx, InitializerContext ctx) { final ColumnDescriptor desc = descriptors[colIdx]; @@ -805,6 +836,79 @@ private FieldDescriptor(GridQueryProperty desc, int fieldIdx) { } } + /** */ + private abstract static class TechnicalDescriptor implements CacheColumnDescriptor { + /** */ + private final String name; + + /** */ + private final Class storageType; + + /** */ + private final int fieldIdx; + + /** */ + private volatile RelDataType logicalType; + + /** */ + private TechnicalDescriptor(String name, Class storageType, int fieldIdx) { + this.name = name; + this.storageType = storageType; + this.fieldIdx = fieldIdx; + } + + /** {@inheritDoc} */ + @Override public boolean field() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean key() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean hasDefaultValue() { + return false; + } + + /** {@inheritDoc} */ + @Override public Object defaultValue() { + throw new AssertionError(); + } + + /** {@inheritDoc} */ + @Override public String name() { + return name; + } + + /** {@inheritDoc} */ + @Override public int fieldIndex() { + return fieldIdx; + } + + /** {@inheritDoc} */ + @Override public RelDataType logicalType(IgniteTypeFactory f) { + if (logicalType == null) { + logicalType = storageType == GridCacheVersion.class + ? f.createTypeWithNullability(f.createJavaType(storageType), true) + : TypeUtils.sqlType(f, storageType, PRECISION_NOT_SPECIFIED, SCALE_NOT_SPECIFIED, true); + } + + return logicalType; + } + + /** {@inheritDoc} */ + @Override public Class storageType() { + return storageType; + } + + /** {@inheritDoc} */ + @Override public void set(Object dst, Object val) { + throw new AssertionError(); + } + } + /** {@inheritDoc} */ @Override public GridQueryTypeDescriptor typeDescription() { return typeDesc; diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/TechnicalColumns.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/TechnicalColumns.java new file mode 100644 index 0000000000000..1884474ef90b1 --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/TechnicalColumns.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.schema; + +/** Technical cache table columns. */ +public final class TechnicalColumns { + /** Field name for row version. */ + public static final String VER_FIELD_NAME = "_VER"; + + /** Field name for row source. */ + public static final String SRC_FIELD_NAME = "_SRC"; + + /** */ + private TechnicalColumns() { + // No-op. + } + + /** */ + public static boolean isTechnicalFieldName(String fieldName) { + return VER_FIELD_NAME.equals(fieldName) || SRC_FIELD_NAME.equals(fieldName); + } + + /** */ + public static boolean isTechnicalFieldNameIgnoreCase(String fieldName) { + return VER_FIELD_NAME.equalsIgnoreCase(fieldName) || SRC_FIELD_NAME.equalsIgnoreCase(fieldName); + } +} \ No newline at end of file diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java new file mode 100644 index 0000000000000..b1e028ee32a6f --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.integration; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.SqlConfiguration; +import org.apache.ignite.configuration.TransactionConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.processors.query.QueryUtils; +import org.apache.ignite.internal.processors.query.calcite.CalciteQueryProcessor; +import org.apache.ignite.internal.processors.query.calcite.exec.ArrayRowHandler; +import org.apache.ignite.internal.processors.query.calcite.exec.ExecutionContext; +import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpIoTracker; +import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpMemoryTracker; +import org.apache.ignite.internal.processors.query.calcite.metadata.ColocationGroup; +import org.apache.ignite.internal.processors.query.calcite.metadata.FragmentDescription; +import org.apache.ignite.internal.processors.query.calcite.metadata.FragmentMapping; +import org.apache.ignite.internal.processors.query.calcite.prepare.BaseQueryContext; +import org.apache.ignite.internal.processors.query.calcite.prepare.MappingQueryContext; +import org.apache.ignite.internal.processors.query.calcite.schema.ColumnDescriptor; +import org.apache.ignite.internal.processors.query.calcite.schema.IgniteCacheTable; +import org.apache.ignite.internal.processors.query.calcite.schema.IgniteIndex; +import org.apache.ignite.internal.processors.query.calcite.schema.TechnicalColumns; +import org.apache.ignite.internal.processors.query.calcite.util.Commons; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.junit.Test; + +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; + +/** Tests technical columns returned by direct table and index scans. */ +public class TechnicalColumnsScanTest extends GridCommonAbstractTest { + /** */ + private IgniteEx node; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setSqlConfiguration(new SqlConfiguration().setQueryEnginesConfiguration( + new CalciteQueryEngineConfiguration().setDefault(true))) + .setTransactionConfiguration(new TransactionConfiguration().setTxAwareQueriesEnabled(true)); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + node = startGrid(0); + + awaitPartitionMapExchange(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** */ + @Test + public void testTableScanReturnsTechnicalColumns() throws Exception { + createAndPopulatePersonTable(); + + IgniteCacheTable tbl = personTable(); + ScanContext scanCtx = scanContext(tbl); + + assertTechnicalColumns(tbl.scan(scanCtx.ectx, scanCtx.grp, requiredColumns(tbl)), tbl); + } + + /** */ + @Test + public void testIndexScanReturnsTechnicalColumns() throws Exception { + createAndPopulatePersonTable(); + + IgniteCacheTable tbl = personTable(); + IgniteIndex idx = tbl.getIndex("AGE_IDX"); + + assertNotNull(idx); + + ScanContext scanCtx = scanContext(tbl); + + assertTechnicalColumns(idx.scan(scanCtx.ectx, scanCtx.grp, null, requiredColumns(tbl)), tbl); + } + + + /** */ + private void createAndPopulatePersonTable() throws Exception { + sql("CREATE TABLE Person (id INT PRIMARY KEY, name VARCHAR, age INT) WITH atomicity=TRANSACTIONAL"); + sql("CREATE INDEX age_idx ON Person(age)"); + + try (Transaction tx = node.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) { + for (int i = 1; i <= 30; i++) + sql("INSERT INTO Person(id, name, age) VALUES (?, ?, ?)", i, personName(i), 20 + i); + + tx.commit(); + } + + awaitPartitionMapExchange(); + } + + /** */ + private void assertTechnicalColumns(Iterable rowsIterable, IgniteCacheTable tbl) throws Exception { + List rows = materialize(rowsIterable); + Set ids = new HashSet<>(); + Integer expSrc = tbl.descriptor().cacheInfo().cacheId(); + + assertEquals(30, rows.size()); + + for (Object[] row : rows) { + assertEquals(3, row.length); + assertTrue(row[0] instanceof Integer); + assertTrue("Unexpected _VER value [val=" + row[1] + ", cls=" + + (row[1] == null ? null : row[1].getClass()) + ']', row[1] instanceof GridCacheVersion); + assertEquals(expSrc, row[2]); + + ids.add((Integer)row[0]); + } + + for (int i = 1; i <= 30; i++) + assertTrue("Missing id: " + i, ids.contains(i)); + } + + /** */ + private List materialize(Iterable rowsIterable) throws Exception { + List rows = new ArrayList<>(); + + try { + for (Object[] row : rowsIterable) + rows.add(row); + } + finally { + if (rowsIterable instanceof AutoCloseable) + ((AutoCloseable)rowsIterable).close(); + } + + return rows; + } + + /** */ + private ImmutableBitSet requiredColumns(IgniteCacheTable tbl) { + return ImmutableBitSet.of( + columnIndex(tbl, "ID"), + columnIndex(tbl, TechnicalColumns.VER_FIELD_NAME), + columnIndex(tbl, TechnicalColumns.SRC_FIELD_NAME) + ); + } + + /** */ + private int columnIndex(IgniteCacheTable tbl, String name) { + ColumnDescriptor desc = tbl.descriptor().columnDescriptor(name); + + assertNotNull(name, desc); + + return desc.fieldIndex(); + } + + /** */ + private ScanContext scanContext(IgniteCacheTable tbl) { + UUID nodeId = node.localNode().id(); + AffinityTopologyVersion topVer = node.context().cache().context().exchange().readyAffinityVersion(); + BaseQueryContext qctx = BaseQueryContext.builder().logger(log).build(); + + ExecutionContext ectx = new ExecutionContext<>( + qctx, + null, + null, + UUID.randomUUID(), + nodeId, + nodeId, + topVer, + new FragmentDescription(0, FragmentMapping.create(nodeId), null, Collections.emptyMap()), + ArrayRowHandler.INSTANCE, + NoOpMemoryTracker.INSTANCE, + NoOpIoTracker.INSTANCE, + 0, + Collections.emptyMap(), + null + ); + + ColocationGroup grp = tbl.colocationGroup(new MappingQueryContext(qctx, nodeId, topVer, null)).finalizeMapping(); + + return new ScanContext(ectx, grp); + } + + /** */ + private IgniteCacheTable personTable() { + CalciteQueryProcessor qryProc = Commons.lookupComponent(node.context(), CalciteQueryProcessor.class); + + return (IgniteCacheTable)qryProc.schemaHolder().schema(QueryUtils.DFLT_SCHEMA).getTable("PERSON"); + } + + /** */ + private List> sql(String sql, Object... args) { + return node.context().query().querySqlFields(new SqlFieldsQuery(sql).setSchema("PUBLIC").setArgs(args), true).getAll(); + } + + /** */ + private String personName(int id) { + switch (id) { + case 1: + return "Alice"; + + case 2: + return "Bob"; + + case 3: + return "Ann"; + + case 4: + return "Carl"; + + case 5: + return "Alex"; + + case 6: + return "Diana"; + + default: + return "Person" + id; + } + } + + /** */ + private static class ScanContext { + /** */ + private final ExecutionContext ectx; + + /** */ + private final ColocationGroup grp; + + /** */ + private ScanContext(ExecutionContext ectx, ColocationGroup grp) { + this.ectx = ectx; + this.grp = grp; + } + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java index f4e123c6a7c40..72caafdfe9255 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java @@ -78,6 +78,7 @@ import org.apache.ignite.internal.processors.query.calcite.integration.SystemViewsIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.TableDdlIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.TableDmlIntegrationTest; +import org.apache.ignite.internal.processors.query.calcite.integration.TechnicalColumnsScanTest; import org.apache.ignite.internal.processors.query.calcite.integration.TimeoutIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.UnnestIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.UnstableTopologyIntegrationTest; @@ -183,6 +184,7 @@ CacheWithInterceptorIntegrationTest.class, TxThreadLockingTest.class, SelectByKeyFieldTest.class, + TechnicalColumnsScanTest.class, }) public class IntegrationTestSuite { } From 8867a32c8be60f16edba9eafa6d6504b2a56ee51 Mon Sep 17 00:00:00 2001 From: Vladislav Pyatkov Date: Mon, 6 Jul 2026 23:24:29 +0300 Subject: [PATCH 2/6] Make technical columns access forbidden in SQL validation. --- .../calcite/prepare/IgniteSqlValidator.java | 40 +++++++++++++++++++ .../query/calcite/util/IgniteResource.java | 4 ++ .../integration/TechnicalColumnsScanTest.java | 31 ++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java index b85e13a0e2c60..75045b2f651f3 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java @@ -247,6 +247,29 @@ private void validateTableModify(SqlNode table) { super.validateSelect(select, targetRowType); } + /** {@inheritDoc} */ + @Override protected void validateOrderList(SqlSelect select) { + super.validateOrderList(select); + + SqlNodeList orderList = select.getOrderList(); + + if (orderList == null) + return; + + for (SqlNode orderItem : orderList) { + SqlNode node = orderItem; + + // Unwrap DESC / NULLS FIRST / NULLS LAST wrappers to get the actual expression. + while (node.getKind() == SqlKind.DESCENDING + || node.getKind() == SqlKind.NULLS_FIRST + || node.getKind() == SqlKind.NULLS_LAST) { + node = ((SqlCall)node).operand(0); + } + + validateTechnicalColumnAccess(node); + } + } + /** {@inheritDoc} */ @Override protected void validateNamespace(SqlValidatorNamespace namespace, RelDataType targetRowType) { SqlValidatorTable table = namespace.getTable(); @@ -561,6 +584,8 @@ private boolean isSystemFieldName(String alias) { /** {@inheritDoc} */ @Override public RelDataType deriveType(SqlValidatorScope scope, SqlNode expr) { + validateTechnicalColumnAccess(expr); + if (expr instanceof SqlDynamicParam) { RelDataType type = deriveDynamicParameterType((SqlDynamicParam)expr, nullType); @@ -571,6 +596,21 @@ private boolean isSystemFieldName(String alias) { return super.deriveType(scope, expr); } + /** */ + private void validateTechnicalColumnAccess(SqlNode expr) { + if (!(expr instanceof SqlIdentifier)) + return; + + SqlIdentifier id = (SqlIdentifier)expr; + + String fieldName = id.names.get(id.names.size() - 1); + + if (id.isStar() || !TechnicalColumns.isTechnicalFieldNameIgnoreCase(fieldName)) + return; + + throw newValidationError(id, IgniteResource.INSTANCE.cannotAccessTechnicalColumn(id.toString())); + } + /** @return A derived type or {@code null} if unable to determine. */ @Nullable private RelDataType deriveDynamicParameterType(SqlDynamicParam node, RelDataType nullValType) { RelDataType type = getValidatedNodeTypeIfKnown(node); diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java index df74d56a91638..a4c5ccd41fc7c 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java @@ -35,6 +35,10 @@ public interface IgniteResource { @Resources.BaseMessage("Cannot update field \"{0}\". You cannot update key, key fields or val field in case the val is a complex type.") Resources.ExInst cannotUpdateField(String field); + /** */ + @Resources.BaseMessage("Cannot access technical column \"{0}\".") + Resources.ExInst cannotAccessTechnicalColumn(String field); + /** */ @Resources.BaseMessage("Illegal aggregate function. {0} is unsupported at the moment.") Resources.ExInst unsupportedAggregationFunction(String a0); diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java index b1e028ee32a6f..3f297490ecbcf 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java @@ -32,6 +32,7 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.processors.query.calcite.CalciteQueryProcessor; import org.apache.ignite.internal.processors.query.calcite.exec.ArrayRowHandler; @@ -48,6 +49,7 @@ import org.apache.ignite.internal.processors.query.calcite.schema.IgniteIndex; import org.apache.ignite.internal.processors.query.calcite.schema.TechnicalColumns; import org.apache.ignite.internal.processors.query.calcite.util.Commons; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.junit.Test; @@ -108,6 +110,35 @@ public void testIndexScanReturnsTechnicalColumns() throws Exception { assertTechnicalColumns(idx.scan(scanCtx.ectx, scanCtx.grp, null, requiredColumns(tbl)), tbl); } + /** */ + @Test + public void testTechnicalColumnsAreHiddenFromSql() throws Exception { + createAndPopulatePersonTable(); + + List> rows = sql("SELECT * FROM Person"); + + assertEquals(30, rows.size()); + assertEquals(3, rows.get(0).size()); + + assertTechnicalColumnAccessForbidden("SELECT _ver FROM Person"); + assertTechnicalColumnAccessForbidden("SELECT _src FROM Person"); + assertTechnicalColumnAccessForbidden("SELECT p._ver FROM Person p"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person WHERE _src IS NOT NULL"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person ORDER BY _ver"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person ORDER BY _ver DESC"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person ORDER BY _src NULLS FIRST"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person GROUP BY _ver"); + assertTechnicalColumnAccessForbidden("SELECT p1.id FROM Person p1 JOIN Person p2 ON p1._ver = p2._ver"); + assertTechnicalColumnAccessForbidden("SELECT CASE WHEN _ver IS NOT NULL THEN 1 ELSE 0 END FROM Person"); + assertTechnicalColumnAccessForbidden("SELECT CAST(_ver AS VARCHAR) FROM Person"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person WHERE (SELECT _ver FROM Person WHERE id = 1) IS NOT NULL"); + } + + /** */ + private void assertTechnicalColumnAccessForbidden(String qry) { + GridTestUtils.assertThrowsAnyCause(log, () -> sql(qry), IgniteSQLException.class, + "Cannot access technical column"); + } /** */ private void createAndPopulatePersonTable() throws Exception { From 8841db1e7643ec7dfca75f34b1bc48b1394dedd6 Mon Sep 17 00:00:00 2001 From: Vladislav Pyatkov Date: Mon, 6 Jul 2026 23:37:28 +0300 Subject: [PATCH 3/6] Fixed code style. --- .../processors/query/calcite/schema/TechnicalColumns.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/TechnicalColumns.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/TechnicalColumns.java index 1884474ef90b1..5ec73c9909181 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/TechnicalColumns.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/TechnicalColumns.java @@ -39,4 +39,4 @@ public static boolean isTechnicalFieldName(String fieldName) { public static boolean isTechnicalFieldNameIgnoreCase(String fieldName) { return VER_FIELD_NAME.equalsIgnoreCase(fieldName) || SRC_FIELD_NAME.equalsIgnoreCase(fieldName); } -} \ No newline at end of file +} From 1e66878f7fc55269b929e5016c16d51a88547d1f Mon Sep 17 00:00:00 2001 From: Vladislav Pyatkov Date: Tue, 7 Jul 2026 14:22:40 +0300 Subject: [PATCH 4/6] Fixed issue with technical columns access in SQL validation and scan operations. --- .../schema/CacheTableDescriptorImpl.java | 34 ++++++- .../integration/TechnicalColumnsScanTest.java | 99 +++++++++++++++++++ 2 files changed, 128 insertions(+), 5 deletions(-) diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java index 17dedb0b56e2e..7e20264c8bc2d 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java @@ -112,6 +112,9 @@ public class CacheTableDescriptorImpl extends NullInitializerExpressionFactory /** */ private final ImmutableBitSet insertFields; + /** Count of non-technical columns used in the UPDATE source SELECT. */ + private final int updateRowFieldCnt; + /** */ private RelDataType tableRowType; @@ -235,6 +238,9 @@ else if (!F.isEmpty(typeDesc.primaryKeyFields())) { this.affFields = ImmutableIntList.copyOf(affFields); this.descriptors = descriptors.toArray(DUMMY); this.descriptorsMap = descriptorsMap; + this.updateRowFieldCnt = (int)descriptors.stream() + .filter(d -> !TechnicalColumns.isTechnicalFieldNameIgnoreCase(d.name())) + .count(); virtualFields.flip(0, descriptors.size()); insertFields = ImmutableBitSet.fromBitSet(virtualFields); @@ -245,6 +251,18 @@ else if (!F.isEmpty(typeDesc.primaryKeyFields())) { return rowType(factory, insertFields); } + /** {@inheritDoc} */ + @Override public RelDataType selectForUpdateRowType(IgniteTypeFactory factory) { + RelDataTypeFactory.Builder b = new RelDataTypeFactory.Builder(factory); + + for (CacheColumnDescriptor desc : descriptors) { + if (!TechnicalColumns.isTechnicalFieldNameIgnoreCase(desc.name())) + b.add(desc.name(), desc.logicalType(factory)); + } + + return b.build(); + } + /** {@inheritDoc} */ @Override public GridCacheContext cacheContext() { return cacheInfo.cacheContext(); @@ -443,7 +461,8 @@ private ModifyTuple updateTuple(Row row, List updateColList, int o Object key = Objects.requireNonNull(hnd.get(offset + QueryUtils.KEY_COL, row)); Object val = clone(Objects.requireNonNull(hnd.get(offset + QueryUtils.VAL_COL, row))); - offset += descriptorsMap.size(); + // New values start after the source fields; compute dynamically to handle both UPDATE and MERGE. + offset = hnd.columnCount(row) - updateColList.size(); for (int i = 0; i < updateColList.size(); i++) { final CacheColumnDescriptor desc = Objects.requireNonNull(descriptorsMap.get(updateColList.get(i))); @@ -473,14 +492,19 @@ private ModifyTuple mergeTuple(Row row, List updateColList, Execut int rowColumnsCnt = hnd.columnCount(row); - if (rowColumnsCnt == descriptors.length) + // An empty update column list unambiguously means there is no WHEN MATCHED clause at all (a MERGE + // statement always has at least one WHEN clause), so the row can only originate from the INSERT + // section. Note: the row width alone can't be used to detect this case, since, depending on the + // number of updated columns, it may coincide with the width of a WHEN MATCHED-only row. + if (updateColList.isEmpty()) return insertTuple(row, ectx); // Only WHEN NOT MATCHED clause in MERGE. - else if (rowColumnsCnt == descriptors.length + updateColList.size()) + else if (rowColumnsCnt == updateRowFieldCnt + updateColList.size()) return updateTuple(row, updateColList, 0, ectx); // Only WHEN MATCHED clause in MERGE. else { // Both WHEN MATCHED and WHEN NOT MATCHED clauses in MERGE. - assert rowColumnsCnt == descriptors.length * 2 + updateColList.size() : "Unexpected columns count: " + - rowColumnsCnt; + // INSERT section has all fields (insertRowType); UPDATE section excludes technical columns. + assert rowColumnsCnt == descriptors.length + updateRowFieldCnt + updateColList.size() : + "Unexpected columns count: " + rowColumnsCnt; int updateOffset = descriptors.length; // Offset of fields for update statement. diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java index 3f297490ecbcf..af7ab0fe9d222 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java @@ -23,14 +23,20 @@ import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.Callable; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheEntry; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.SqlConfiguration; import org.apache.ignite.configuration.TransactionConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.CacheEntryImplEx; +import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryUtils; @@ -49,6 +55,7 @@ import org.apache.ignite.internal.processors.query.calcite.schema.IgniteIndex; import org.apache.ignite.internal.processors.query.calcite.schema.TechnicalColumns; import org.apache.ignite.internal.processors.query.calcite.util.Commons; +import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -95,6 +102,72 @@ public void testTableScanReturnsTechnicalColumns() throws Exception { assertTechnicalColumns(tbl.scan(scanCtx.ectx, scanCtx.grp, requiredColumns(tbl)), tbl); } + /** */ + @Test + @SuppressWarnings("unchecked") + public void testScannedTechnicalColumnsCanLockTxEntries() throws Exception { + createAndPopulatePersonTable(); + + IgniteCacheTable tbl = personTable(); + ScanContext scanCtx = scanContext(tbl); + List rows = materialize(tbl.scan(scanCtx.ectx, scanCtx.grp, lockRequiredColumns(tbl))); + List> entries = new ArrayList<>(); + Integer expSrc = tbl.descriptor().cacheInfo().cacheId(); + + assertEquals(30, rows.size()); + + for (Object[] row : rows) { + assertEquals(4, row.length); + assertTrue("Unexpected _VER value [val=" + row[2] + ", cls=" + + (row[2] == null ? null : row[2].getClass()) + ']', row[2] instanceof GridCacheVersion); + assertEquals(expSrc, row[3]); + + entries.add(new CacheEntryImplEx<>(row[0], row[1], (GridCacheVersion)row[2])); + } + + try (Transaction tx = node.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) { + assertTrue(node.cache(tbl.descriptor().cacheInfo().name()).unwrap(IgniteCacheProxy.class) + .internalProxy().lockTxEntries(entries, 5_000)); + + checkInaccessInOtherTx(); + + sql("UPDATE Person SET age = 42 WHERE id = 2"); + + tx.commit(); + } + + List> rowsAfterUpdate = sql("SELECT id, name, age FROM Person WHERE id = 2"); + + assertEquals(1, rowsAfterUpdate.size()); + assertEquals(personName(2), rowsAfterUpdate.get(0).get(1)); + assertEquals(42, rowsAfterUpdate.get(0).get(2)); + } + + /** + * Checks that another transaction cannot access the cache. + * + * @throws IgniteCheckedException If failed. + */ + private void checkInaccessInOtherTx() throws IgniteCheckedException { + IgniteInternalFuture accessFut = GridTestUtils.runAsync(new Callable() { + @Override public Void call() { + try (Transaction tx = node.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 500, 1)) { + sql("UPDATE Person SET name = 'Charley' WHERE id = 2"); + + tx.commit(); + } + + return null; + } + }); + + GridTestUtils.assertThrowsWithCause(new Callable() { + @Override public Object call() throws Exception { + return accessFut.get(10_000); + } + }, IgniteTxTimeoutCheckedException.class); + } + /** */ @Test public void testIndexScanReturnsTechnicalColumns() throws Exception { @@ -132,6 +205,22 @@ public void testTechnicalColumnsAreHiddenFromSql() throws Exception { assertTechnicalColumnAccessForbidden("SELECT CASE WHEN _ver IS NOT NULL THEN 1 ELSE 0 END FROM Person"); assertTechnicalColumnAccessForbidden("SELECT CAST(_ver AS VARCHAR) FROM Person"); assertTechnicalColumnAccessForbidden("SELECT id FROM Person WHERE (SELECT _ver FROM Person WHERE id = 1) IS NOT NULL"); + + // MERGE: technical columns must be forbidden in all clause positions. + assertTechnicalColumnAccessForbidden( + "MERGE INTO Person " + + "USING (SELECT id, _ver FROM Person) AS src ON (Person.id = src.id) " + + "WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (src.id, 'x', 1)"); + + assertTechnicalColumnAccessForbidden( + "MERGE INTO Person " + + "USING (SELECT 100 AS id) AS src ON (Person._ver IS NOT NULL AND Person.id = src.id) " + + "WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (src.id, 'x', 1)"); + + assertTechnicalColumnAccessForbidden( + "MERGE INTO Person " + + "USING (SELECT 1 AS id) AS src ON (Person.id = src.id) " + + "WHEN MATCHED THEN UPDATE SET name = CAST(Person._ver AS VARCHAR)"); } /** */ @@ -202,6 +291,16 @@ private ImmutableBitSet requiredColumns(IgniteCacheTable tbl) { ); } + /** */ + private ImmutableBitSet lockRequiredColumns(IgniteCacheTable tbl) { + return ImmutableBitSet.of( + columnIndex(tbl, QueryUtils.KEY_FIELD_NAME), + columnIndex(tbl, QueryUtils.VAL_FIELD_NAME), + columnIndex(tbl, TechnicalColumns.VER_FIELD_NAME), + columnIndex(tbl, TechnicalColumns.SRC_FIELD_NAME) + ); + } + /** */ private int columnIndex(IgniteCacheTable tbl, String name) { ColumnDescriptor desc = tbl.descriptor().columnDescriptor(name); From 10dd99dd54c5429535f2a0b8f1cd8e4bfc4d9da0 Mon Sep 17 00:00:00 2001 From: Vladislav Pyatkov Date: Tue, 7 Jul 2026 21:55:34 +0300 Subject: [PATCH 5/6] Fix technical column access in SQL --- .../calcite/prepare/IgniteSqlValidator.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java index 75045b2f651f3..10db05c1babbe 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java @@ -435,18 +435,26 @@ private SqlNode rewriteTableToQuery(SqlNode from) { SelectScope scope, boolean includeSysVars ) { - if (!includeSysVars && exp.getKind() == SqlKind.IDENTIFIER && isSystemFieldName(deriveAlias(exp, 0))) { - SqlQualified qualified = scope.fullyQualify((SqlIdentifier)exp); + if (!includeSysVars && exp.getKind() == SqlKind.IDENTIFIER) { + String alias = deriveAlias(exp, 0); - if (qualified.namespace == null) + // Technical columns are never exposed via SELECT *. + if (TechnicalColumns.isTechnicalFieldNameIgnoreCase(alias)) return; - if (qualified.namespace.getTable() != null) { - // If child is table and has only system fields, expand star to these fields. - // Otherwise, expand star to non-system fields only. - for (RelDataTypeField fld : qualified.namespace.getRowType().getFieldList()) { - if (!isSystemField(fld)) - return; + if (isSystemFieldName(alias)) { + SqlQualified qualified = scope.fullyQualify((SqlIdentifier)exp); + + if (qualified.namespace == null) + return; + + if (qualified.namespace.getTable() != null) { + // If child is table and has only system fields, expand star to these fields. + // Otherwise, expand star to non-system fields only. + for (RelDataTypeField fld : qualified.namespace.getRowType().getFieldList()) { + if (!isSystemField(fld)) + return; + } } } } From 835b119cfadb7a9b183ad55305e707f0ebdbbb38 Mon Sep 17 00:00:00 2001 From: Vladislav Pyatkov Date: Wed, 8 Jul 2026 10:12:49 +0300 Subject: [PATCH 6/6] Exclude technical columns from SQL validation and scan operations. --- .../query/calcite/schema/CacheTableDescriptorImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java index 7e20264c8bc2d..8ff3fb42dc059 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java @@ -545,8 +545,10 @@ private ModifyTuple deleteTuple(Row row, ExecutionContext ectx) { RelDataTypeFactory.Builder b = new RelDataTypeFactory.Builder(factory); if (usedColumns == null) { - for (int i = 0; i < descriptors.length; i++) - b.add(descriptors[i].name(), descriptors[i].logicalType(factory)); + for (int i = 0; i < descriptors.length; i++) { + if (!isTechnicalColumn(descriptors[i].name())) + b.add(descriptors[i].name(), descriptors[i].logicalType(factory)); + } return tableRowType = b.build(); }