diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java index 8d25cf1c249a0..250cc28eb1a5c 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java @@ -246,6 +246,29 @@ private void validateTableModify(SqlNode table) { super.validateSelect(select, targetRowType); } + /** {@inheritDoc} */ + @Override protected void validateOrderList(SqlSelect select) { + super.validateOrderList(select); + + SqlNodeList orderList = select.getOrderList(); + + if (orderList == null) + return; + + for (SqlNode orderItem : orderList) { + SqlNode node = orderItem; + + // Unwrap DESC / NULLS FIRST / NULLS LAST wrappers to get the actual expression. + while (node.getKind() == SqlKind.DESCENDING + || node.getKind() == SqlKind.NULLS_FIRST + || node.getKind() == SqlKind.NULLS_LAST) { + node = ((SqlCall)node).operand(0); + } + + validateTechnicalColumnAccess(node); + } + } + /** {@inheritDoc} */ @Override protected void validateNamespace(SqlValidatorNamespace namespace, RelDataType targetRowType) { SqlValidatorTable table = namespace.getTable(); @@ -293,7 +316,7 @@ else if (n instanceof SqlDynamicParam) { if (call.getKind() == SqlKind.AS) { final String alias = deriveAlias(call, 0); - if (isSystemFieldName(alias)) + if (QueryUtils.isReservedFieldName(alias)) throw newValidationError(call, IgniteResource.INSTANCE.illegalAlias(alias)); } else if (call.getKind() == SqlKind.CAST) { @@ -411,18 +434,26 @@ private SqlNode rewriteTableToQuery(SqlNode from) { SelectScope scope, boolean includeSysVars ) { - if (!includeSysVars && exp.getKind() == SqlKind.IDENTIFIER && isSystemFieldName(deriveAlias(exp, 0))) { - SqlQualified qualified = scope.fullyQualify((SqlIdentifier)exp); + if (!includeSysVars && exp.getKind() == SqlKind.IDENTIFIER) { + String alias = deriveAlias(exp, 0); - if (qualified.namespace == null) + // Technical columns are never exposed via SELECT *. + if (QueryUtils.isTechnicalFieldNameIgnoreCase(alias)) return; - if (qualified.namespace.getTable() != null) { - // If child is table and has only system fields, expand star to these fields. - // Otherwise, expand star to non-system fields only. - for (RelDataTypeField fld : qualified.namespace.getRowType().getFieldList()) { - if (!isSystemField(fld)) - return; + if (QueryUtils.isReservedFieldName(alias)) { + SqlQualified qualified = scope.fullyQualify((SqlIdentifier)exp); + + if (qualified.namespace == null) + return; + + if (qualified.namespace.getTable() != null) { + // If child is table and has only system fields, expand star to these fields. + // Otherwise, expand star to non-system fields only. + for (RelDataTypeField fld : qualified.namespace.getRowType().getFieldList()) { + if (!isSystemField(fld)) + return; + } } } } @@ -432,7 +463,7 @@ private SqlNode rewriteTableToQuery(SqlNode from) { /** {@inheritDoc} */ @Override public boolean isSystemField(RelDataTypeField field) { - return isSystemFieldName(field.getName()); + return QueryUtils.isReservedFieldName(field.getName()); } /** */ @@ -551,14 +582,10 @@ private IgniteTypeFactory typeFactory() { return (IgniteTypeFactory)typeFactory; } - /** */ - private boolean isSystemFieldName(String alias) { - return QueryUtils.KEY_FIELD_NAME.equalsIgnoreCase(alias) - || QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(alias); - } - /** {@inheritDoc} */ @Override public RelDataType deriveType(SqlValidatorScope scope, SqlNode expr) { + validateTechnicalColumnAccess(expr); + if (expr instanceof SqlDynamicParam) { RelDataType type = deriveDynamicParameterType((SqlDynamicParam)expr, nullType); @@ -569,6 +596,21 @@ private boolean isSystemFieldName(String alias) { return super.deriveType(scope, expr); } + /** */ + private void validateTechnicalColumnAccess(SqlNode expr) { + if (!(expr instanceof SqlIdentifier)) + return; + + SqlIdentifier id = (SqlIdentifier)expr; + + String fieldName = id.names.get(id.names.size() - 1); + + if (id.isStar() || !QueryUtils.isTechnicalFieldNameIgnoreCase(fieldName)) + return; + + throw newValidationError(id, IgniteResource.INSTANCE.cannotAccessTechnicalColumn(id.toString())); + } + /** @return A derived type or {@code null} if unable to determine. */ @Nullable private RelDataType deriveDynamicParameterType(SqlDynamicParam node, RelDataType nullValType) { RelDataType type = getValidatedNodeTypeIfKnown(node); diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java index 2cd164bc81570..077bed92f98e3 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java @@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.query.GridQueryProperty; import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor; import org.apache.ignite.internal.processors.query.QueryUtils; @@ -111,6 +112,9 @@ public class CacheTableDescriptorImpl extends NullInitializerExpressionFactory /** */ private final ImmutableBitSet insertFields; + /** Count of non-technical columns used in MERGE source SELECT sections. */ + private final int mergeSourceRowFieldCnt; + /** */ private RelDataType tableRowType; @@ -123,7 +127,7 @@ public CacheTableDescriptorImpl(GridCacheContextInfo cacheInfo, GridQueryT Set fields = this.typeDesc.fields().keySet(); - List descriptors = new ArrayList<>(fields.size() + 2); + List descriptors = new ArrayList<>(fields.size() + 4); // A _key/_val field is virtual in case there is an alias or a property(es) mapped to the _key/_val field. BitSet virtualFields = new BitSet(); @@ -177,6 +181,28 @@ else if (Objects.equals(field, typeDesc.valueFieldAlias())) { } } + virtualFields.set(descriptors.size()); + descriptors.add(new TechnicalDescriptor(QueryUtils.VER_FIELD_NAME, GridCacheVersion.class, descriptors.size()) { + @Override public Object value(ExecutionContext ectx, GridCacheContext cctx, CacheDataRow src) + throws IgniteCheckedException { + GridCacheVersion ver = src.version(); + + if (ver != null) + return ver; + + CacheDataRow row = cctx.offheap().read(cctx, src.key()); + + return row == null ? null : row.version(); + } + }); + + virtualFields.set(descriptors.size()); + descriptors.add(new TechnicalDescriptor(QueryUtils.SRC_FIELD_NAME, Integer.class, descriptors.size()) { + @Override public Object value(ExecutionContext ectx, GridCacheContext cctx, CacheDataRow src) { + return cctx.cacheId(); + } + }); + Map descriptorsMap = U.newHashMap(descriptors.size()); for (CacheColumnDescriptor descriptor : descriptors) descriptorsMap.put(descriptor.name(), descriptor); @@ -212,6 +238,9 @@ else if (!F.isEmpty(typeDesc.primaryKeyFields())) { this.affFields = ImmutableIntList.copyOf(affFields); this.descriptors = descriptors.toArray(DUMMY); this.descriptorsMap = descriptorsMap; + this.mergeSourceRowFieldCnt = (int)descriptors.stream() + .filter(d -> !QueryUtils.isTechnicalFieldNameIgnoreCase(d.name())) + .count(); virtualFields.flip(0, descriptors.size()); insertFields = ImmutableBitSet.fromBitSet(virtualFields); @@ -222,6 +251,18 @@ else if (!F.isEmpty(typeDesc.primaryKeyFields())) { return rowType(factory, insertFields); } + /** {@inheritDoc} */ + @Override public RelDataType selectForUpdateRowType(IgniteTypeFactory factory) { + RelDataTypeFactory.Builder b = new RelDataTypeFactory.Builder(factory); + + for (CacheColumnDescriptor desc : descriptors) { + if (!QueryUtils.isTechnicalFieldNameIgnoreCase(desc.name())) + b.add(desc.name(), desc.logicalType(factory)); + } + + return b.build(); + } + /** {@inheritDoc} */ @Override public GridCacheContext cacheContext() { return cacheInfo.cacheContext(); @@ -278,6 +319,9 @@ else if (affFields.isEmpty()) @Override public boolean isUpdateAllowed(RelOptTable tbl, int colIdx) { final CacheColumnDescriptor desc = descriptors[colIdx]; + if (isTechnicalColumn(desc.name())) + return false; + return !desc.key() && (desc.field() || QueryUtils.isSqlType(desc.storageType())); } @@ -289,6 +333,11 @@ else if (affFields.isEmpty()) return super.generationStrategy(tbl, colIdx); } + /** */ + private static boolean isTechnicalColumn(String fieldName) { + return QueryUtils.isTechnicalFieldName(fieldName); + } + /** {@inheritDoc} */ @Override public RexNode newColumnDefaultValue(RelOptTable tbl, int colIdx, InitializerContext ctx) { final ColumnDescriptor desc = descriptors[colIdx]; @@ -382,9 +431,12 @@ private Object insertVal(Row row, ExecutionContext ectx) throws Ignit for (int i = 2; i < descriptors.length; i++) { final CacheColumnDescriptor desc = descriptors[i]; + if (!desc.field() || desc.key()) + continue; + Object fieldVal = hnd.get(i, row); - if (desc.field() && !desc.key() && fieldVal != null) + if (fieldVal != null) desc.set(val, TypeUtils.fromInternal(ectx, fieldVal, desc.storageType())); } } @@ -412,7 +464,8 @@ private ModifyTuple updateTuple(Row row, List updateColList, int o Object key = Objects.requireNonNull(hnd.get(offset + QueryUtils.KEY_COL, row)); Object val = clone(Objects.requireNonNull(hnd.get(offset + QueryUtils.VAL_COL, row))); - offset += descriptorsMap.size(); + // New values start after the source fields; compute dynamically to handle both UPDATE and MERGE. + offset = hnd.columnCount(row) - updateColList.size(); for (int i = 0; i < updateColList.size(); i++) { final CacheColumnDescriptor desc = Objects.requireNonNull(descriptorsMap.get(updateColList.get(i))); @@ -442,16 +495,21 @@ private ModifyTuple mergeTuple(Row row, List updateColList, Execut int rowColumnsCnt = hnd.columnCount(row); - if (rowColumnsCnt == descriptors.length) + // An empty update column list unambiguously means there is no WHEN MATCHED clause at all (a MERGE + // statement always has at least one WHEN clause), so the row can only originate from the INSERT + // section. Note: the row width alone can't be used to detect this case, since, depending on the + // number of updated columns, it may coincide with the width of a WHEN MATCHED-only row. + if (updateColList.isEmpty()) return insertTuple(row, ectx); // Only WHEN NOT MATCHED clause in MERGE. - else if (rowColumnsCnt == descriptors.length + updateColList.size()) + else if (rowColumnsCnt == mergeSourceRowFieldCnt + updateColList.size()) return updateTuple(row, updateColList, 0, ectx); // Only WHEN MATCHED clause in MERGE. else { // Both WHEN MATCHED and WHEN NOT MATCHED clauses in MERGE. - assert rowColumnsCnt == descriptors.length * 2 + updateColList.size() : "Unexpected columns count: " + - rowColumnsCnt; + // INSERT and UPDATE source sections both exclude technical columns. + assert rowColumnsCnt == mergeSourceRowFieldCnt * 2 + updateColList.size() : + "Unexpected columns count: " + rowColumnsCnt; - int updateOffset = descriptors.length; // Offset of fields for update statement. + int updateOffset = mergeSourceRowFieldCnt; // Offset of fields for update statement. if (hnd.get(updateOffset + QueryUtils.KEY_COL, row) != null) return updateTuple(row, updateColList, updateOffset, ectx); @@ -490,8 +548,10 @@ private ModifyTuple deleteTuple(Row row, ExecutionContext ectx) { RelDataTypeFactory.Builder b = new RelDataTypeFactory.Builder(factory); if (usedColumns == null) { - for (int i = 0; i < descriptors.length; i++) - b.add(descriptors[i].name(), descriptors[i].logicalType(factory)); + for (int i = 0; i < descriptors.length; i++) { + if (!isTechnicalColumn(descriptors[i].name())) + b.add(descriptors[i].name(), descriptors[i].logicalType(factory)); + } return tableRowType = b.build(); } @@ -805,6 +865,79 @@ private FieldDescriptor(GridQueryProperty desc, int fieldIdx) { } } + /** */ + private abstract static class TechnicalDescriptor implements CacheColumnDescriptor { + /** */ + private final String name; + + /** */ + private final Class storageType; + + /** */ + private final int fieldIdx; + + /** */ + private volatile RelDataType logicalType; + + /** */ + private TechnicalDescriptor(String name, Class storageType, int fieldIdx) { + this.name = name; + this.storageType = storageType; + this.fieldIdx = fieldIdx; + } + + /** {@inheritDoc} */ + @Override public boolean field() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean key() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean hasDefaultValue() { + return false; + } + + /** {@inheritDoc} */ + @Override public Object defaultValue() { + throw new AssertionError(); + } + + /** {@inheritDoc} */ + @Override public String name() { + return name; + } + + /** {@inheritDoc} */ + @Override public int fieldIndex() { + return fieldIdx; + } + + /** {@inheritDoc} */ + @Override public RelDataType logicalType(IgniteTypeFactory f) { + if (logicalType == null) { + logicalType = storageType == GridCacheVersion.class + ? f.createTypeWithNullability(f.createJavaType(storageType), true) + : TypeUtils.sqlType(f, storageType, PRECISION_NOT_SPECIFIED, SCALE_NOT_SPECIFIED, true); + } + + return logicalType; + } + + /** {@inheritDoc} */ + @Override public Class storageType() { + return storageType; + } + + /** {@inheritDoc} */ + @Override public void set(Object dst, Object val) { + throw new AssertionError(); + } + } + /** {@inheritDoc} */ @Override public GridQueryTypeDescriptor typeDescription() { return typeDesc; diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java index df74d56a91638..a4c5ccd41fc7c 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java @@ -35,6 +35,10 @@ public interface IgniteResource { @Resources.BaseMessage("Cannot update field \"{0}\". You cannot update key, key fields or val field in case the val is a complex type.") Resources.ExInst cannotUpdateField(String field); + /** */ + @Resources.BaseMessage("Cannot access technical column \"{0}\".") + Resources.ExInst cannotAccessTechnicalColumn(String field); + /** */ @Resources.BaseMessage("Illegal aggregate function. {0} is unsupported at the moment.") Resources.ExInst unsupportedAggregationFunction(String a0); diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java new file mode 100644 index 0000000000000..392de0c0370ae --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java @@ -0,0 +1,452 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.integration; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Callable; +import javax.cache.CacheException; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheEntry; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.SqlConfiguration; +import org.apache.ignite.configuration.TransactionConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.CacheEntryImplEx; +import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.processors.query.IgniteSQLException; +import org.apache.ignite.internal.processors.query.QueryUtils; +import org.apache.ignite.internal.processors.query.calcite.CalciteQueryProcessor; +import org.apache.ignite.internal.processors.query.calcite.exec.ArrayRowHandler; +import org.apache.ignite.internal.processors.query.calcite.exec.ExecutionContext; +import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpIoTracker; +import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpMemoryTracker; +import org.apache.ignite.internal.processors.query.calcite.metadata.ColocationGroup; +import org.apache.ignite.internal.processors.query.calcite.metadata.FragmentDescription; +import org.apache.ignite.internal.processors.query.calcite.metadata.FragmentMapping; +import org.apache.ignite.internal.processors.query.calcite.prepare.BaseQueryContext; +import org.apache.ignite.internal.processors.query.calcite.prepare.MappingQueryContext; +import org.apache.ignite.internal.processors.query.calcite.schema.ColumnDescriptor; +import org.apache.ignite.internal.processors.query.calcite.schema.IgniteCacheTable; +import org.apache.ignite.internal.processors.query.calcite.schema.IgniteIndex; +import org.apache.ignite.internal.processors.query.calcite.util.Commons; +import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.junit.Test; + +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; + +/** Tests technical columns returned by direct table and index scans. */ +public class TechnicalColumnsScanTest extends GridCommonAbstractTest { + /** */ + private IgniteEx node; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setSqlConfiguration(new SqlConfiguration().setQueryEnginesConfiguration( + new CalciteQueryEngineConfiguration().setDefault(true))) + .setTransactionConfiguration(new TransactionConfiguration().setTxAwareQueriesEnabled(true)); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + node = startGrid(0); + + awaitPartitionMapExchange(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** */ + @Test + public void testTableScanReturnsTechnicalColumns() throws Exception { + createAndPopulatePersonTable(); + + IgniteCacheTable tbl = personTable(); + ScanContext scanCtx = scanContext(tbl); + + assertTechnicalColumns(tbl.scan(scanCtx.ectx, scanCtx.grp, requiredColumns(tbl)), tbl); + } + + /** */ + @Test + @SuppressWarnings("unchecked") + public void testScannedTechnicalColumnsCanLockTxEntries() throws Exception { + createAndPopulatePersonTable(); + + IgniteCacheTable tbl = personTable(); + ScanContext scanCtx = scanContext(tbl); + List rows = materialize(tbl.scan(scanCtx.ectx, scanCtx.grp, lockRequiredColumns(tbl))); + List> entries = new ArrayList<>(); + Integer expSrc = tbl.descriptor().cacheInfo().cacheId(); + + assertEquals(30, rows.size()); + + for (Object[] row : rows) { + assertEquals(4, row.length); + assertTrue("Unexpected _VER value [val=" + row[2] + ", cls=" + + (row[2] == null ? null : row[2].getClass()) + ']', row[2] instanceof GridCacheVersion); + assertEquals(expSrc, row[3]); + + entries.add(new CacheEntryImplEx<>(row[0], row[1], (GridCacheVersion)row[2])); + } + + try (Transaction tx = node.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) { + assertTrue(node.cache(tbl.descriptor().cacheInfo().name()).unwrap(IgniteCacheProxy.class) + .internalProxy().lockTxEntries(entries, 5_000)); + + checkInaccessInOtherTx(); + + sql("UPDATE Person SET age = 42 WHERE id = 2"); + + tx.commit(); + } + + List> rowsAfterUpdate = sql("SELECT id, name, age FROM Person WHERE id = 2"); + + assertEquals(1, rowsAfterUpdate.size()); + assertEquals(personName(2), rowsAfterUpdate.get(0).get(1)); + assertEquals(42, rowsAfterUpdate.get(0).get(2)); + } + + /** + * Checks that another transaction cannot access the cache. + * + * @throws IgniteCheckedException If failed. + */ + private void checkInaccessInOtherTx() throws IgniteCheckedException { + IgniteInternalFuture accessFut = GridTestUtils.runAsync(new Callable() { + @Override public Void call() { + try (Transaction tx = node.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 500, 1)) { + sql("UPDATE Person SET name = 'Charley' WHERE id = 2"); + + tx.commit(); + } + + return null; + } + }); + + GridTestUtils.assertThrowsWithCause(new Callable() { + @Override public Object call() throws Exception { + return accessFut.get(10_000); + } + }, IgniteTxTimeoutCheckedException.class); + } + + /** */ + @Test + public void testIndexScanReturnsTechnicalColumns() throws Exception { + createAndPopulatePersonTable(); + + IgniteCacheTable tbl = personTable(); + IgniteIndex idx = tbl.getIndex("AGE_IDX"); + + assertNotNull(idx); + + ScanContext scanCtx = scanContext(tbl); + + assertTechnicalColumns(idx.scan(scanCtx.ectx, scanCtx.grp, null, requiredColumns(tbl)), tbl); + } + + /** */ + @Test + public void testCannotCreateTableWithTechnicalColumnNames() { + assertTechnicalColumnCreateForbidden("CREATE TABLE PersonVer (id INT PRIMARY KEY, _ver INT)", + QueryUtils.VER_FIELD_NAME); + assertTechnicalColumnCreateForbidden("CREATE TABLE PersonSrc (id INT PRIMARY KEY, _src INT)", + QueryUtils.SRC_FIELD_NAME); + } + + /** */ + @Test + public void testCannotAddTechnicalColumnNames() throws Exception { + createAndPopulatePersonTable(); + + assertTechnicalColumnAddForbidden("ALTER TABLE Person ADD COLUMN _ver INT", + QueryUtils.VER_FIELD_NAME); + assertTechnicalColumnAddForbidden("ALTER TABLE Person ADD COLUMN _src INT", + QueryUtils.SRC_FIELD_NAME); + } + + /** */ + @Test + public void testTechnicalColumnsAreHiddenFromSql() throws Exception { + createAndPopulatePersonTable(); + + List> rows = sql("SELECT * FROM Person"); + + assertEquals(30, rows.size()); + assertEquals(3, rows.get(0).size()); + + assertTechnicalColumnAccessForbidden("SELECT _ver FROM Person"); + assertTechnicalColumnAccessForbidden("SELECT _src FROM Person"); + assertTechnicalColumnAccessForbidden("SELECT p._ver FROM Person p"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person WHERE _src IS NOT NULL"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person ORDER BY _ver"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person ORDER BY _ver DESC"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person ORDER BY _src NULLS FIRST"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person GROUP BY _ver"); + assertTechnicalColumnAccessForbidden("SELECT p1.id FROM Person p1 JOIN Person p2 ON p1._ver = p2._ver"); + assertTechnicalColumnAccessForbidden("SELECT CASE WHEN _ver IS NOT NULL THEN 1 ELSE 0 END FROM Person"); + assertTechnicalColumnAccessForbidden("SELECT CAST(_ver AS VARCHAR) FROM Person"); + assertTechnicalColumnAccessForbidden("SELECT id FROM Person WHERE (SELECT _ver FROM Person WHERE id = 1) IS NOT NULL"); + + // MERGE: technical columns must not be exposed in all clause positions. + assertTechnicalColumnAccessForbidden( + "MERGE INTO Person " + + "USING (SELECT id, _ver FROM Person) AS src ON (Person.id = src.id) " + + "WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (src.id, 'x', 1)"); + + assertTechnicalColumnAccessForbidden( + "MERGE INTO Person " + + "USING (SELECT 100 AS id) AS src ON (Person._ver IS NOT NULL AND Person.id = src.id) " + + "WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (src.id, 'x', 1)"); + + assertTechnicalColumnAccessForbidden( + "MERGE INTO Person " + + "USING (SELECT 1 AS id) AS src ON (Person.id = src.id) " + + "WHEN MATCHED THEN UPDATE SET name = CAST(Person._ver AS VARCHAR)"); + } + + /** */ + private void assertTechnicalColumnCreateForbidden(String qry, String colName) { + String expMsg = "Name '" + colName + "' is reserved and cannot be used as a field name"; + + try { + sql(qry); + + fail("Exception is expected"); + } + catch (CacheException e) { + assertTrue("Expected reserved field name exception was not found: " + e, + hasCauseOrSuppressed(e, IgniteCheckedException.class, expMsg)); + } + } + + /** */ + private boolean hasCauseOrSuppressed(Throwable t, Class cls, String msg) { + if (t == null) + return false; + + if (cls.isAssignableFrom(t.getClass()) && t.getMessage() != null && t.getMessage().contains(msg)) + return true; + + if (hasCauseOrSuppressed(t.getCause(), cls, msg)) + return true; + + for (Throwable suppressed : t.getSuppressed()) { + if (hasCauseOrSuppressed(suppressed, cls, msg)) + return true; + } + + return false; + } + + /** */ + private void assertTechnicalColumnAddForbidden(String qry, String colName) { + GridTestUtils.assertThrowsAnyCause(log, () -> sql(qry), IgniteSQLException.class, + "Column already exists: " + colName); + } + + /** */ + private void assertTechnicalColumnAccessForbidden(String qry) { + GridTestUtils.assertThrowsAnyCause(log, () -> sql(qry), IgniteSQLException.class, "not found"); + } + + /** */ + private void createAndPopulatePersonTable() throws Exception { + sql("CREATE TABLE Person (id INT PRIMARY KEY, name VARCHAR, age INT) WITH atomicity=TRANSACTIONAL"); + sql("CREATE INDEX age_idx ON Person(age)"); + + try (Transaction tx = node.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) { + for (int i = 1; i <= 30; i++) + sql("INSERT INTO Person(id, name, age) VALUES (?, ?, ?)", i, personName(i), 20 + i); + + tx.commit(); + } + + awaitPartitionMapExchange(); + } + + /** */ + private void assertTechnicalColumns(Iterable rowsIterable, IgniteCacheTable tbl) throws Exception { + List rows = materialize(rowsIterable); + Set ids = new HashSet<>(); + Integer expSrc = tbl.descriptor().cacheInfo().cacheId(); + + assertEquals(30, rows.size()); + + for (Object[] row : rows) { + assertEquals(3, row.length); + assertTrue(row[0] instanceof Integer); + assertTrue("Unexpected _VER value [val=" + row[1] + ", cls=" + + (row[1] == null ? null : row[1].getClass()) + ']', row[1] instanceof GridCacheVersion); + assertEquals(expSrc, row[2]); + + ids.add((Integer)row[0]); + } + + for (int i = 1; i <= 30; i++) + assertTrue("Missing id: " + i, ids.contains(i)); + } + + /** */ + private List materialize(Iterable rowsIterable) throws Exception { + List rows = new ArrayList<>(); + + try { + for (Object[] row : rowsIterable) + rows.add(row); + } + finally { + if (rowsIterable instanceof AutoCloseable) + ((AutoCloseable)rowsIterable).close(); + } + + return rows; + } + + /** */ + private ImmutableBitSet requiredColumns(IgniteCacheTable tbl) { + return ImmutableBitSet.of( + columnIndex(tbl, "ID"), + columnIndex(tbl, QueryUtils.VER_FIELD_NAME), + columnIndex(tbl, QueryUtils.SRC_FIELD_NAME) + ); + } + + /** */ + private ImmutableBitSet lockRequiredColumns(IgniteCacheTable tbl) { + return ImmutableBitSet.of( + columnIndex(tbl, QueryUtils.KEY_FIELD_NAME), + columnIndex(tbl, QueryUtils.VAL_FIELD_NAME), + columnIndex(tbl, QueryUtils.VER_FIELD_NAME), + columnIndex(tbl, QueryUtils.SRC_FIELD_NAME) + ); + } + + /** */ + private int columnIndex(IgniteCacheTable tbl, String name) { + ColumnDescriptor desc = tbl.descriptor().columnDescriptor(name); + + assertNotNull(name, desc); + + return desc.fieldIndex(); + } + + /** */ + private ScanContext scanContext(IgniteCacheTable tbl) { + UUID nodeId = node.localNode().id(); + AffinityTopologyVersion topVer = node.context().cache().context().exchange().readyAffinityVersion(); + BaseQueryContext qctx = BaseQueryContext.builder().logger(log).build(); + + ExecutionContext ectx = new ExecutionContext<>( + qctx, + null, + null, + UUID.randomUUID(), + nodeId, + nodeId, + topVer, + new FragmentDescription(0, FragmentMapping.create(nodeId), null, Collections.emptyMap()), + ArrayRowHandler.INSTANCE, + NoOpMemoryTracker.INSTANCE, + NoOpIoTracker.INSTANCE, + 0, + Collections.emptyMap(), + null + ); + + ColocationGroup grp = tbl.colocationGroup(new MappingQueryContext(qctx, nodeId, topVer, null)).finalizeMapping(); + + return new ScanContext(ectx, grp); + } + + /** */ + private IgniteCacheTable personTable() { + CalciteQueryProcessor qryProc = Commons.lookupComponent(node.context(), CalciteQueryProcessor.class); + + return (IgniteCacheTable)qryProc.schemaHolder().schema(QueryUtils.DFLT_SCHEMA).getTable("PERSON"); + } + + /** */ + private List> sql(String sql, Object... args) { + return node.context().query().querySqlFields(new SqlFieldsQuery(sql).setSchema("PUBLIC").setArgs(args), true).getAll(); + } + + /** */ + private String personName(int id) { + switch (id) { + case 1: + return "Alice"; + + case 2: + return "Bob"; + + case 3: + return "Ann"; + + case 4: + return "Carl"; + + case 5: + return "Alex"; + + case 6: + return "Diana"; + + default: + return "Person" + id; + } + } + + /** */ + private static class ScanContext { + /** */ + private final ExecutionContext ectx; + + /** */ + private final ColocationGroup grp; + + /** */ + private ScanContext(ExecutionContext ectx, ColocationGroup grp) { + this.ectx = ectx; + this.grp = grp; + } + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java index f4e123c6a7c40..72caafdfe9255 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java @@ -78,6 +78,7 @@ import org.apache.ignite.internal.processors.query.calcite.integration.SystemViewsIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.TableDdlIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.TableDmlIntegrationTest; +import org.apache.ignite.internal.processors.query.calcite.integration.TechnicalColumnsScanTest; import org.apache.ignite.internal.processors.query.calcite.integration.TimeoutIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.UnnestIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.UnstableTopologyIntegrationTest; @@ -183,6 +184,7 @@ CacheWithInterceptorIntegrationTest.class, TxThreadLockingTest.class, SelectByKeyFieldTest.class, + TechnicalColumnsScanTest.class, }) public class IntegrationTestSuite { } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryTypeDescriptorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryTypeDescriptorImpl.java index 7b917e69a4d30..becc2c4078e38 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryTypeDescriptorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryTypeDescriptorImpl.java @@ -411,7 +411,8 @@ else if (F.isEmpty(keyFieldAlias()) || !usedProps.contains(keyFieldAlias())) * @return {@code True} if exists. */ public boolean hasField(String field) { - return props.containsKey(field) || QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(field); + return props.containsKey(field) || QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(field) || + QueryUtils.isTechnicalFieldNameIgnoreCase(field); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java index 332a7d1ab2780..6ee712e806a6b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java @@ -111,6 +111,12 @@ public class QueryUtils { /** Field name for value. */ public static final String VAL_FIELD_NAME = CommonUtils.VAL_FIELD_NAME; + /** Field name for row version. */ + public static final String VER_FIELD_NAME = "_VER"; + + /** Field name for row source. */ + public static final String SRC_FIELD_NAME = "_SRC"; + /** Well-known template name for PARTITIONED cache. */ public static final String TEMPLATE_PARTITIONED = "PARTITIONED"; @@ -1514,6 +1520,22 @@ public static SchemaOperationException validateDropColumn(GridQueryTypeDescripto return null; } + /** */ + public static boolean isReservedFieldName(String fieldName) { + return KEY_FIELD_NAME.equalsIgnoreCase(fieldName) || VAL_FIELD_NAME.equalsIgnoreCase(fieldName) || + isTechnicalFieldNameIgnoreCase(fieldName); + } + + /** */ + public static boolean isTechnicalFieldName(String fieldName) { + return VER_FIELD_NAME.equals(fieldName) || SRC_FIELD_NAME.equals(fieldName); + } + + /** */ + public static boolean isTechnicalFieldNameIgnoreCase(String fieldName) { + return VER_FIELD_NAME.equalsIgnoreCase(fieldName) || SRC_FIELD_NAME.equalsIgnoreCase(fieldName); + } + /** * Returns true if the exception is triggered by query cancel. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java index 750ff09dee523..09039ae51add4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java @@ -86,6 +86,7 @@ import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName; import static org.apache.ignite.internal.processors.query.QueryUtils.KEY_FIELD_NAME; import static org.apache.ignite.internal.processors.query.QueryUtils.VAL_FIELD_NAME; +import static org.apache.ignite.internal.processors.query.QueryUtils.isReservedFieldName; import static org.apache.ignite.internal.processors.query.QueryUtils.matches; /** @@ -411,7 +412,7 @@ private static void validateTypeDescriptor(GridQueryTypeDescriptor type) throws String ptrn = "Name ''{0}'' is reserved and cannot be used as a field name [type=" + type.name() + "]"; for (String name : names) { - if (name.equalsIgnoreCase(KEY_FIELD_NAME) || name.equalsIgnoreCase(VAL_FIELD_NAME)) + if (isReservedFieldName(name)) throw new IgniteCheckedException(MessageFormat.format(ptrn, name)); } }