From 0753c3a0b040a69927f4b37c8dfe7699fc7bbb74 Mon Sep 17 00:00:00 2001 From: Vivek1106-04 Date: Fri, 24 Jul 2026 21:47:59 +0530 Subject: [PATCH 1/2] [SPARK-58013][PS] Fix loc setitem dropping writes for reordered columns on pandas 3 The pandas 3 CoW branch in `LocIndexerLike.__setitem__` returned early (no-op) when a scalar was assigned to a list of columns not in the frame's internal order. pandas 3 actually applies such writes, so the guard silently dropped valid assignments. Remove it; both column orders now use the normal path. The change is behind `pandas >= 3.0.0`, so pandas 2 is unaffected. --- python/pyspark/pandas/indexing.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/python/pyspark/pandas/indexing.py b/python/pyspark/pandas/indexing.py index 236118a6d6136..34d470376dd3b 100644 --- a/python/pyspark/pandas/indexing.py +++ b/python/pyspark/pandas/indexing.py @@ -763,30 +763,6 @@ def __setitem__(self, key: Any, value: Any) -> None: if isinstance(value, Series): value = value.spark.column else: - if ( - # Only apply this behavior for pandas 3+, where CoW semantics changed. - LooseVersion(pd.__version__) >= "3.0.0" - # Only for multi-column assignment (single-column assignment is unaffected). - and len(selected_column_labels) > 1 - # Column selector must be list-like (e.g. ["shield", "max_speed"]), not scalar label access. - and is_list_like(cols_sel) - # Excludes string/bytes (single label), tuple (e.g. MultiIndex label), - # and slice selectors; keeps this narrowly on explicit column lists. - and not isinstance(cols_sel, (str, bytes, tuple, slice)) - # Only trigger when cached/anchored Series exist on the frame, - # matching the problematic case where views were materialized before assignment. - and hasattr(self._psdf_or_psser, "_psseries") - ): - selected_column_labels_set = set(selected_column_labels) - selected_labels_in_internal_order = [ - label - for label in self._internal.column_labels - if label in selected_column_labels_set - ] - if selected_column_labels != selected_labels_in_internal_order: - # If requested columns are in different order than the DataFrame's internal order, - # it returns early (no-op), matching pandas 3 behavior for that edge case. - return value = F.lit(value) new_data_spark_columns = [] From 16bb35949d40515bbbd39741eda7f3a449bdd982 Mon Sep 17 00:00:00 2001 From: Vivek1106-04 Date: Sat, 25 Jul 2026 12:37:14 +0530 Subject: [PATCH 2/2] [SPARK-58013][PS] Revert unused unpack name to _ in loc setitem --- python/pyspark/pandas/indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/pandas/indexing.py b/python/pyspark/pandas/indexing.py index 34d470376dd3b..579143692c84b 100644 --- a/python/pyspark/pandas/indexing.py +++ b/python/pyspark/pandas/indexing.py @@ -741,7 +741,7 @@ def __setitem__(self, key: Any, value: Any) -> None: cond, limit, remaining_index = self._select_rows(rows_sel) missing_keys: List[Name] = [] ( - selected_column_labels, + _, data_spark_columns, _, _,