diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index ba5de2b41c..693ab33eaa 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -1343,7 +1343,7 @@ const DBSearchPageFiltersComponent = ({ // Coerce dot-form Map sub-keys (LogAttributes.foo) into bracket form // (LogAttributes['foo']) before handing them to ClickHouse. Bracket // form is the canonical SQL key produced by mergePath; dot form ends - // up in filterState after setFilterValue's parseKeyPath().join('.') + // up in filterState after setFilterValue's columnKeyToDotPath() // normalization or after a Lucene URL round-trip, and ClickHouse // cannot resolve it as map access. const sqlKey = toClickHouseKeyExpression(key); diff --git a/packages/app/src/components/DBSearchPageFilters/utils.ts b/packages/app/src/components/DBSearchPageFilters/utils.ts index 7ccdd44874..bcfd99575f 100644 --- a/packages/app/src/components/DBSearchPageFilters/utils.ts +++ b/packages/app/src/components/DBSearchPageFilters/utils.ts @@ -2,6 +2,7 @@ import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import type { FilterState } from '@hyperdx/common-utils/dist/filters'; +import { isMapColumn } from '@hyperdx/common-utils/dist/guards'; import { mergePath } from '@/utils'; @@ -22,12 +23,12 @@ export function parseMapFieldName( key: string, ): { baseName: string; propertyPath: string } | null { const cleanKey = cleanClickHouseExpression(key); - const path = parseKeyPath(cleanKey); + const col = parseKeyPath(cleanKey); - if (path.length >= 2) { + if (isMapColumn(col)) { return { - baseName: path[0], - propertyPath: path.slice(1).join('.'), + baseName: col.column, + propertyPath: col.key, }; } @@ -48,7 +49,7 @@ export function parseMapFieldName( // Bracket-form keys (e.g. LogAttributes['time']) are the canonical SQL form // produced by mergePath for Map columns, while dot-form keys // (e.g. LogAttributes.time) are what setFilterValue stores after its -// parseKeyPath().join('.') normalization and what parseLuceneFilter returns +// columnKeyToDotPath() normalization and what parseLuceneFilter returns // on URL load. Same logical field, different raw string. function isBracketFormMapKey(key: string): boolean { return key.includes("['") || key.includes('["'); @@ -123,7 +124,7 @@ export function groupFacetsByBaseName( // Look up a filterState entry by either bracket-form or dot-form map sub-key. // Bracket form is the canonical SQL key used in facet results; dot form is -// what setFilterValue stores after its parseKeyPath().join('.') normalization +// what setFilterValue stores after its columnKeyToDotPath() normalization // and what parseLuceneFilter restores from a Lucene URL round-trip. Reads // need to tolerate either so the user's selection still resolves regardless // of which form `child.key` carries after groupFacetsByBaseName's merge. diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx index 1e1b10ad17..3efeed7497 100644 --- a/packages/app/src/hooks/useAutoCompleteOptions.tsx +++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx @@ -1,5 +1,6 @@ import { useMemo } from 'react'; import { + columnKeyToDotPath, Field, parseKeyPath, TableConnection, @@ -256,7 +257,7 @@ export function useAutoCompleteOptions( if (!keyValues || keyValues.length === 0) return []; return keyValues.flatMap(kv => { - const fieldName = parseKeyPath(kv.key).join('.'); + const fieldName = columnKeyToDotPath(parseKeyPath(kv.key)); return kv.value.flatMap((v: string | Record) => { if (typeof v === 'object' && v !== null) { // Map columns can return objects like { 'service.name': 'frontend' } diff --git a/packages/app/src/hooks/useDashboardFilters.tsx b/packages/app/src/hooks/useDashboardFilters.tsx index 093cf84254..ff0f82f88c 100644 --- a/packages/app/src/hooks/useDashboardFilters.tsx +++ b/packages/app/src/hooks/useDashboardFilters.tsx @@ -1,6 +1,9 @@ import { useCallback, useEffect, useMemo, useRef } from 'react'; import { useQueryState } from 'nuqs'; -import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; +import { + columnKeyToDotPath, + parseKeyPath, +} from '@hyperdx/common-utils/dist/core/metadata'; import { FilterState, filtersToQuery, @@ -24,7 +27,7 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { const { filters: filterValues } = parseQuery(prev ?? []); // Normalize the expression to dot notation so it matches the keys // returned by parseQuery (which converts bracket notation to dots). - const key = parseKeyPath(expression).join('.'); + const key = columnKeyToDotPath(parseKeyPath(expression)); if (values.length === 0) { delete filterValues[key]; } else { @@ -53,7 +56,7 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { // Build a normalized lookup so bracket-notation expressions // (e.g. SpanAttributes['k8s.pod.name']) match the dot-notation keys // returned by parseLuceneFilter (e.g. SpanAttributes.k8s.pod.name). - const normalizeKey = (k: string) => parseKeyPath(k).join('.'); + const normalizeKey = (k: string) => columnKeyToDotPath(parseKeyPath(k)); const normalizedParsed = new Map( Object.entries(parsedFilters).map(([k, v]) => [normalizeKey(k), v]), ); diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index 23ec18ba42..9985043ef9 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -1,6 +1,9 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import produce from 'immer'; -import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; +import { + columnKeyToDotPath, + parseKeyPath, +} from '@hyperdx/common-utils/dist/core/metadata'; import { coerceBooleanValue, type FilterState, @@ -444,7 +447,7 @@ export const useSearchPageFilterState = ({ ) => { // Normalize bracket notation to dot notation so the key matches // what parseQuery returns after a Lucene round-trip. - const key = parseKeyPath(property).join('.'); + const key = columnKeyToDotPath(parseKeyPath(property)); // Normalize "true"/"false" strings to booleans so the value matches // what parseLuceneFilter returns after a Lucene round-trip. const normalizedValue = coerceBooleanValue(value); @@ -491,7 +494,7 @@ export const useSearchPageFilterState = ({ const setFilterRange = useCallback( (property: string, range: { min: number; max: number }) => { - const key = parseKeyPath(property).join('.'); + const key = columnKeyToDotPath(parseKeyPath(property)); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { if (!draft[key]) { @@ -508,7 +511,7 @@ export const useSearchPageFilterState = ({ const clearFilter = useCallback( (property: string) => { - const key = parseKeyPath(property).join('.'); + const key = columnKeyToDotPath(parseKeyPath(property)); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { delete draft[key]; diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index bbd40781ef..b56cf13eda 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -3,7 +3,12 @@ import { Metadata, MetadataCache, parseKeyPath } from '../core/metadata'; import * as renderChartConfigModule from '../core/renderChartConfig'; import { timeFilterExpr } from '../core/renderChartConfig'; import { isBuilderChartConfig } from '../guards'; -import { BuilderChartConfigWithDateRange, SourceKind, TSource } from '../types'; +import { + BuilderChartConfigWithDateRange, + NATIVE_COLUMN, + SourceKind, + TSource, +} from '../types'; // Mock ClickhouseClient const mockClickhouseClient = { @@ -1438,37 +1443,41 @@ describe('Metadata', () => { describe('parseKeyPath', () => { it('parses single-quoted bracket notation', () => { - expect(parseKeyPath("ResourceAttributes['service.name']")).toEqual([ - 'ResourceAttributes', - 'service.name', - ]); + expect(parseKeyPath("ResourceAttributes['service.name']")).toEqual({ + column: 'ResourceAttributes', + key: 'service.name', + }); }); it('parses double-quoted bracket notation', () => { - expect(parseKeyPath('ResourceAttributes["service.name"]')).toEqual([ - 'ResourceAttributes', - 'service.name', - ]); + expect(parseKeyPath('ResourceAttributes["service.name"]')).toEqual({ + column: 'ResourceAttributes', + key: 'service.name', + }); }); - it('returns single-element path for native columns', () => { - expect(parseKeyPath('ServiceName')).toEqual(['ServiceName']); + it('returns native column for top-level columns', () => { + expect(parseKeyPath('ServiceName')).toEqual({ + column: NATIVE_COLUMN, + key: 'ServiceName', + }); }); it('handles keys with dots in the map key', () => { - expect(parseKeyPath("SpanAttributes['http.request.method']")).toEqual([ - 'SpanAttributes', - 'http.request.method', - ]); + expect(parseKeyPath("SpanAttributes['http.request.method']")).toEqual({ + column: 'SpanAttributes', + key: 'http.request.method', + }); }); - it('returns single-element path for empty string', () => { - expect(parseKeyPath('')).toEqual(['']); + it('returns native column for empty string', () => { + expect(parseKeyPath('')).toEqual({ column: NATIVE_COLUMN, key: '' }); }); it('does not parse incomplete bracket notation', () => { - expect(parseKeyPath("ResourceAttributes['service.name")).toEqual([ - "ResourceAttributes['service.name", - ]); + expect(parseKeyPath("ResourceAttributes['service.name")).toEqual({ + column: NATIVE_COLUMN, + key: "ResourceAttributes['service.name", + }); }); }); diff --git a/packages/common-utils/src/core/eventDeltas.ts b/packages/common-utils/src/core/eventDeltas.ts index 060b40d3bc..1576799454 100644 --- a/packages/common-utils/src/core/eventDeltas.ts +++ b/packages/common-utils/src/core/eventDeltas.ts @@ -22,7 +22,7 @@ */ export function flattenData(data: Record): Record { const result: Record = {}; - // eslint-disable-next-line security/detect-object-injection -- prop is built from known object keys via recursion, not user input + function recurse(cur: Record, prop: string) { if (Object(cur) !== cur) { result[prop] = cur; // eslint-disable-line security/detect-object-injection diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index a80feb1fad..1e3181803a 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -16,10 +16,11 @@ import { renderChartConfig, timeFilterExpr } from '@/core/renderChartConfig'; import type { BuilderChartConfig, BuilderChartConfigWithDateRange, + ColumnKey, MetadataMaterializedViews, TSource, } from '@/types'; -import { isLogSource, isTraceSource, SourceKind } from '@/types'; +import { isLogSource, isTraceSource, NATIVE_COLUMN, SourceKind } from '@/types'; import { ClickHouseVersion, parseClickHouseVersion } from './clickhouseVersion'; import { @@ -1350,14 +1351,14 @@ export class Metadata { // Parse all keys into (rollupColumn, rollupKey) pairs const parsed = keyExpressions.map(keyExpr => { - const path = parseKeyPath(keyExpr); - const isMapKey = path.length >= 2; + const col = parseKeyPath(keyExpr); + const isNative = col.column === NATIVE_COLUMN; return { keyExpression: keyExpr, - rollupColumn: isMapKey ? path[0] : 'NativeColumn', - rollupKey: isMapKey ? path[1] : path[0], - column: path[0], - mapKey: isMapKey ? path[1] : undefined, + rollupColumn: col.column, + rollupKey: col.key, + column: isNative ? col.key : col.column, + mapKey: isNative ? undefined : col.key, }; }); @@ -1546,7 +1547,7 @@ export class Metadata { WHERE Value != '' AND ${timeFilter} GROUP BY ColumnIdentifier, Key - ORDER BY ColumnIdentifier = 'NativeColumn' DESC, ColumnIdentifier, Key + ORDER BY ColumnIdentifier = '${NATIVE_COLUMN}' DESC, ColumnIdentifier, Key ${limitClause} `; @@ -1569,7 +1570,7 @@ export class Metadata { return rows.map(row => { const keyExpr = - row.ColumnIdentifier === 'NativeColumn' + row.ColumnIdentifier === NATIVE_COLUMN ? row.Key : `${row.ColumnIdentifier}['${row.Key}']`; return { key: keyExpr, value: row.Values }; @@ -1768,20 +1769,40 @@ export type Field = { }; /** - * Parses a bracket-notation key string into a path array. - * e.g. `ResourceAttributes['service.name']` → `['ResourceAttributes', 'service.name']` - * `ServiceName` → `['ServiceName']` + * Parses a bracket-notation key string into a {@link ColumnKey}. + * e.g. `ResourceAttributes['service.name']` → `{ column: 'ResourceAttributes', key: 'service.name' }` + * `ServiceName` → `{ column: NATIVE_COLUMN, key: 'ServiceName' }` */ -export function parseKeyPath(key: string): string[] { +export function parseKeyPath(key: string): ColumnKey { const singleIdx = key.indexOf("['"); if (singleIdx !== -1 && key.endsWith("']")) { - return [key.slice(0, singleIdx), key.slice(singleIdx + 2, -2)]; + return { + column: key.slice(0, singleIdx), + key: key.slice(singleIdx + 2, -2), + }; } const doubleIdx = key.indexOf('["'); if (doubleIdx !== -1 && key.endsWith('"]')) { - return [key.slice(0, doubleIdx), key.slice(doubleIdx + 2, -2)]; + return { + column: key.slice(0, doubleIdx), + key: key.slice(doubleIdx + 2, -2), + }; } - return [key]; + return { + column: NATIVE_COLUMN, + key, + }; +} + +/** + * Renders a {@link ColumnKey} as a dotted path string. Inverse of + * `parseKeyPath` for the dot-form normalization used to match keys across + * Lucene round-trips and filter-state lookups. + * e.g. `{ column: 'ResourceAttributes', key: 'service.name' }` → `ResourceAttributes.service.name` + * `{ column: NATIVE_COLUMN, key: 'ServiceName' }` → `ServiceName` + */ +export function columnKeyToDotPath(col: ColumnKey): string { + return col.column === NATIVE_COLUMN ? col.key : `${col.column}.${col.key}`; } // Describes a table and potentially related views diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index e5ded218a4..b52bb8e3ce 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -1,6 +1,6 @@ import lucene from '@hyperdx/lucene'; -import { parseKeyPath } from '@/core/metadata'; +import { columnKeyToDotPath, parseKeyPath } from '@/core/metadata'; import { decodeSpecialTokens, isBinaryAST, @@ -27,7 +27,7 @@ const escapeLuceneQuotedTerm = (s: string) => { /** * Escape backslashes and colons so the field name survives Lucene parsing. * Map sub-keys can legitimately contain `:` (e.g. `LogAttributes['foo:bar']` - * normalizes to `LogAttributes.foo:bar` via parseKeyPath().join('.')), and + * normalizes to `LogAttributes.foo:bar` via columnKeyToDotPath()), and * `:` is the Lucene field/value separator. Backslashes are escaped first so * the inserted `\:` survives `encodeSpecialTokens`' `\\` → backslash-literal * substitution; the encoder's matching `\:` → HDX_COLON rule then makes the @@ -47,7 +47,9 @@ export const filtersToQuery = (filters: FilterState): Filter[] => { ) .flatMap(([key, values]) => { const conditions: Filter[] = []; - const luceneField = escapeLuceneFieldName(parseKeyPath(key).join('.')); + const luceneField = escapeLuceneFieldName( + columnKeyToDotPath(parseKeyPath(key)), + ); if (values.included.size > 0) { const terms = Array.from(values.included).map( diff --git a/packages/common-utils/src/guards.ts b/packages/common-utils/src/guards.ts index 059be3473b..f4913559bc 100644 --- a/packages/common-utils/src/guards.ts +++ b/packages/common-utils/src/guards.ts @@ -3,7 +3,10 @@ import { BuilderSavedChartConfig, ChartConfig, ChartConfigWithOptDateRange, + ColumnKey, DisplayType, + MapColumnKey, + NATIVE_COLUMN, PromqlChartConfig, PromqlSavedChartConfig, RawSqlChartConfig, @@ -76,6 +79,10 @@ export function isBuilderSavedChartConfig( ); } +export function isMapColumn(column: ColumnKey): column is MapColumnKey { + return column.column !== NATIVE_COLUMN; +} + /** * Returns true when a display type semantically requires a data source to be * configured. Currently Markdown is the only display type that does not need a diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index fb195d016b..f9451b5cd0 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -22,6 +22,8 @@ import { } from '@/core/utils'; import { UseTextIndex } from '@/types'; +import { isMapColumn } from './guards'; + /** Max number of tokens to pass to hasAllTokens(), which supports up to 64 tokens as of ClickHouse v25.12. */ const HAS_ALL_TOKENS_CHUNK_SIZE = 50; @@ -48,9 +50,9 @@ export function parse(query: string): lucene.AST { } function buildMapContains(mapField: string) { - const path = parseKeyPath(mapField); - if (path.length < 2) return undefined; - return SqlString.format('mapContains(??, ?)', [path[0], path[1]]); + const col = parseKeyPath(mapField); + if (!isMapColumn(col)) return undefined; + return SqlString.format('mapContains(??, ?)', [col.column, col.key]); } /** Strip whitespace and backtick-quoting from a ClickHouse expression for comparison */ diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 64b5cde975..84b4bc5a13 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1897,3 +1897,45 @@ export const MeApiResponseSchema = z.object({ }); export type MeApiResponse = z.infer; + +/** + * Sentinel value for {@link ColumnKey} `column` indicating a top-level + * (non-Map/JSON) column. When `column` equals this, `key` carries the + * native column name itself. + */ +export const NATIVE_COLUMN = 'NativeColumn' as const; + +/** + * A reference to a top-level native column. + * `key` is the native column name (e.g. `'ServiceName'`). + */ +export const NativeColumnKeySchema = z.object({ + column: z.literal(NATIVE_COLUMN), + key: z.string(), +}); + +/** + * A reference to a value inside a Map or JSON column. + * `column` is the Map/JSON column name (e.g. `'ResourceAttributes'`). + * `key` is the key/path within it (e.g. `'service.name'`). + */ +export const MapColumnKeySchema = z.object({ + column: z.string().refine(s => s !== NATIVE_COLUMN, { + message: `column must not equal "${NATIVE_COLUMN}"; use NativeColumnKeySchema for native columns`, + }), + key: z.string(), +}); + +/** + * Discriminated union over `column`: + * - `column === NATIVE_COLUMN` → {@link NativeColumnKey} + * - otherwise → {@link MapColumnKey} + */ +export const ColumnKeySchema = z.union([ + NativeColumnKeySchema, + MapColumnKeySchema, +]); + +export type NativeColumnKey = z.infer; +export type MapColumnKey = z.infer; +export type ColumnKey = z.infer;