Skip to content

[core] Introduce truncatedString for representations of large collections #7000

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 1 commit into from
Jan 12, 2026
Merged

[core] Introduce truncatedString for representations of large collections #7000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions paimon-api/src/main/java/org/apache/paimon/utils/StringUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.paimon.utils;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
Expand All @@ -45,6 +46,9 @@ public class StringUtils {
/** The empty String {@code ""}. */
public static final String EMPTY = "";

/** Default maximum number of fields for truncated string representation. */
public static final int DEFAULT_MAX_FIELDS = 25;

/**
* Checks if the string is null, empty, or contains only whitespace characters. A whitespace
* character is defined via {@link Character#isWhitespace(char)}.
Expand Down Expand Up @@ -562,4 +566,46 @@ public static boolean isOpenBracket(char c) {
public static boolean isCloseBracket(char c) {
return c == ']' || c == '}' || c == ')';
}

/**
* Converts a sequence to a string with truncation if it exceeds the maximum number of fields.
* This is useful for limiting the size of string representations of large collections.
*
* @param lst the collection to convert to string
* @param start the prefix string
* @param sep the separator between elements
* @param end the suffix string
* @param maxFields the maximum number of fields to include before truncation
* @return the truncated string representation
*/
public static String truncatedString(
Collection<?> lst, String start, String sep, String end, int maxFields) {
boolean truncated = lst.size() > maxFields;
int numFields = truncated ? Math.max(0, maxFields - 1) : lst.size();

StringBuilder builder = new StringBuilder();
builder.append(start);

Iterator<?> iterator = lst.iterator();
for (int i = 0; i < numFields; i++) {
if (i > 0) {
builder.append(sep);
}
builder.append(iterator.next());
}

if (truncated) {
builder.append(sep)
.append("... ")
.append(lst.size() - numFields)
.append(" more fields");
}

builder.append(end);
return builder.toString();
}

public static String truncatedString(Collection<?> lst, String start, String sep, String end) {
return truncatedString(lst, start, sep, end, DEFAULT_MAX_FIELDS);
}
}
8 changes: 5 additions & 3 deletions paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.paimon.io.DataInputViewStreamWrapper;
import org.apache.paimon.io.DataOutputViewStreamWrapper;
import org.apache.paimon.types.DataType;
import org.apache.paimon.utils.StringUtils;

import java.io.IOException;
import java.io.ObjectInputStream;
Expand Down Expand Up @@ -119,12 +120,13 @@ public <T> T visit(PredicateVisitor<T> visitor) {
@Override
public String toString() {
String literalsStr;
if (literals == null || literals.isEmpty()) {
int literalsSize = literals == null ? 0 : literals.size();
if (literalsSize == 0) {
literalsStr = "";
} else if (literals.size() == 1) {
} else if (literalsSize == 1) {
literalsStr = Objects.toString(literals.get(0));
} else {
literalsStr = literals.toString();
literalsStr = StringUtils.truncatedString(literals, "[", ", ", "]");
}
return literalsStr.isEmpty()
? function + "(" + fieldName() + ")"
Expand Down
2 changes: 1 addition & 1 deletion paimon-common/src/main/java/org/apache/paimon/predicate/PredicateBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ private Predicate leaf(LeafFunction function, Transform transform) {
public Predicate in(int idx, List<Object> literals) {
// In the IN predicate, 20 literals are critical for performance.
// If there are more than 20 literals, the performance will decrease.
if (literals.size() > 20 || literals.size() == 0) {
if (literals.size() > 20 || literals.isEmpty()) {
DataField field = rowType.getFields().get(idx);
return new LeafPredicate(In.INSTANCE, field.type(), idx, field.name(), literals);
}
Expand Down
13 changes: 13 additions & 0 deletions paimon-common/src/test/java/org/apache/paimon/predicate/PredicateTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -712,4 +712,17 @@ public void testPredicateToString() {
.isEqualTo(
"NotIn(f0, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21])");
}

@Test
public void testPredicateToStringWithManyFields() {
PredicateBuilder builder = new PredicateBuilder(RowType.of(new IntType()));
List<Object> literals = new ArrayList<>();
for (int i = 1; i <= 100; i++) {
literals.add(i);
}
Predicate p = builder.in(0, literals);
assertThat(p.toString())
.isEqualTo(
"In(f0, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, ... 76 more fields])");
}
}
95 changes: 95 additions & 0 deletions paimon-common/src/test/java/org/apache/paimon/utils/StringUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,101 @@ void testIsNumericInvalidNumbers(String input) {
}
}

@Nested
class TruncatedStringTests {

@Test
void testTruncatedStringWithinMaxFields() {
List<String> items = Arrays.asList("a", "b", "c");
String result = StringUtils.truncatedString(items, "[", ", ", "]", 5);
assertThat(result).isEqualTo("[a, b, c]");
}

@Test
void testTruncatedStringExactlyMaxFields() {
List<String> items = Arrays.asList("a", "b", "c");
String result = StringUtils.truncatedString(items, "[", ", ", "]", 3);
assertThat(result).isEqualTo("[a, b, c]");
}

@Test
void testTruncatedStringExceedsMaxFields() {
List<String> items = Arrays.asList("a", "b", "c", "d", "e");
String result = StringUtils.truncatedString(items, "[", ", ", "]", 3);
assertThat(result).isEqualTo("[a, b, ... 3 more fields]");
}

@Test
void testTruncatedStringExceedsMaxFieldsWithSeparator() {
List<Integer> items = Arrays.asList(1, 2, 3, 4, 5, 6);
String result = StringUtils.truncatedString(items, "(", "-", ")", 4);
assertThat(result).isEqualTo("(1-2-3-... 3 more fields)");
}

@Test
void testTruncatedStringEmptyCollection() {
List<String> items = Arrays.asList();
String result = StringUtils.truncatedString(items, "[", ", ", "]", 3);
assertThat(result).isEqualTo("[]");
}

@Test
void testTruncatedStringSingleElement() {
List<String> items = Arrays.asList("only");
String result = StringUtils.truncatedString(items, "[", ", ", "]", 5);
assertThat(result).isEqualTo("[only]");
}

@Test
void testTruncatedStringMaxFieldsZero() {
List<String> items = Arrays.asList("a", "b", "c");
String result = StringUtils.truncatedString(items, "[", ", ", "]", 0);
assertThat(result).isEqualTo("[, ... 3 more fields]");
}

@Test
void testTruncatedStringMaxFieldsOne() {
List<String> items = Arrays.asList("a", "b", "c", "d");
String result = StringUtils.truncatedString(items, "[", ", ", "]", 1);
assertThat(result).isEqualTo("[, ... 4 more fields]");
}

@Test
void testTruncatedStringLargeCollection() {
List<Integer> items = Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
String result = StringUtils.truncatedString(items, "{", ", ", "}", 5);
assertThat(result).isEqualTo("{1, 2, 3, 4, ... 6 more fields}");
}

@Test
void testTruncatedStringWithEmptyStrings() {
List<String> items = Arrays.asList("", "a", "", "b", "");
String result = StringUtils.truncatedString(items, "[", "|", "]", 3);
assertThat(result).isEqualTo("[|a|... 3 more fields]");
}

@Test
void testTruncatedStringWithNullElements() {
List<String> items = Arrays.asList("a", null, "b", "c");
String result = StringUtils.truncatedString(items, "[", ", ", "]", 3);
assertThat(result).isEqualTo("[a, null, ... 2 more fields]");
}

@Test
void testTruncatedStringWithCustomDelimiters() {
List<String> items = Arrays.asList("apple", "banana", "cherry", "date");
String result = StringUtils.truncatedString(items, "<", " | ", ">", 3);
assertThat(result).isEqualTo("<apple | banana | ... 2 more fields>");
}

@Test
void testTruncatedStringWithEmptyDelimiters() {
List<String> items = Arrays.asList("a", "b", "c");
String result = StringUtils.truncatedString(items, "", "", "", 5);
assertThat(result).isEqualTo("abc");
}
}

@Nested
class EdgeCaseTests {

Expand Down