Polishment

Author: vdusek

scripts/_utils.py

@@ -27,7 +27,6 @@
     (re.compile(r'\bSynchronous\b'), 'Asynchronous'),
     (re.compile(r'Retry a function'), 'Retry an async function'),
     (re.compile(r'Function to retry'), 'Async function to retry'),
-    (re.compile(r'returned page also supports iteration: `for'), 'returned page also supports iteration: `async for'),
 ]
 """Patterns for converting sync docstrings to async docstrings."""
 

src/apify_client/_pagination.py

@@ -14,23 +14,6 @@ class HasItems(Protocol[T]):
     items: list[T]
 
 
-def _min_for_limit_param(a: int | None, b: int | None) -> int | None:
-    """Return minimum of two limit parameters, treating `None` or `0` as infinity.
-
-    The Apify API treats `0` as no limit for the `limit` parameter, so `0` here means infinity.
-    Returns `None` when both inputs represent infinity.
-    """
-    if a == 0:
-        a = None
-    if b == 0:
-        b = None
-    if a is None:
-        return b
-    if b is None:
-        return a
-    return min(a, b)
-
-
 def get_items_iterator(
     callback: Callable[..., HasItems[T]],
     *,
@@ -40,13 +23,13 @@ def get_items_iterator(
 ) -> Iterator[T]:
     """Yield individual items from offset-based paginated API responses.
 
-    The `callback` is invoked lazily to fetch each page from the API. It must accept `limit` and
-    `offset` keyword arguments and return an object whose `items` attribute is a list. If the
-    object also exposes a `count` attribute, it is used for offset bookkeeping (the Apify API's
-    `count` reflects items scanned, which can exceed items returned when filters are applied).
+    The `callback` is invoked lazily to fetch each page from the API. It must accept `limit` and `offset` keyword
+    arguments and return an object whose `items` attribute is a list. If the object also exposes a `count` attribute, it
+    is used for offset bookkeeping (the Apify API's `count` reflects items scanned, which can exceed items returned when
+    filters are applied).
 
-    Iteration stops when a page returns no items or when the user-requested `limit` is reached.
-    The `total` field is intentionally not consulted, because it can change between calls.
+    Iteration stops when a page returns no items or when the user-requested `limit` is reached. The `total` field is
+    intentionally not consulted, because it can change between calls.
 
     Args:
         callback: Function returning a single page of items.
@@ -61,9 +44,7 @@ def get_items_iterator(
 
     while True:
         current_page = callback(
-            limit=effective_chunk
-            if not initial_limit
-            else _min_for_limit_param(initial_limit - fetched_items, effective_chunk),
+            limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
             offset=initial_offset + fetched_items,
         )
         yield from current_page.items
@@ -92,9 +73,7 @@ async def get_items_iterator_async(
 
     while True:
         current_page = await callback(
-            limit=effective_chunk
-            if not initial_limit
-            else _min_for_limit_param(initial_limit - fetched_items, effective_chunk),
+            limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
             offset=initial_offset + fetched_items,
         )
         for item in current_page.items:
@@ -133,13 +112,11 @@ def get_cursor_iterator(
 ) -> Iterator[Request] | Iterator[KeyValueStoreKey]:
     """Yield individual items from cursor-paginated API responses.
 
-    Each page is expected to expose `items` and `next_<cursor_param>`; iteration ends when a
-    page returns no items, the next cursor is `None`, or the user-requested `limit` is reached.
+    Each page is expected to expose `items` and a next-cursor field; iteration ends when a page returns no items, the
+    next cursor is `None`, or the user-requested `limit` is reached.
 
     Args:
-        callback: Function returning a single page of items. Receives the cursor as a kwarg
-            named after `cursor_param` and a `limit` kwarg.
-        cursor_param: Name of the cursor query-parameter (e.g. `cursor` or `exclusive_start_key`).
+        callback: Function returning a single page of items. Receives `cursor` and `limit` kwargs.
         cursor: Value of the cursor for the first request, or `None` to start from the beginning.
         limit: Maximum total number of items to yield across all pages.
         chunk_size: Maximum number of items requested per API call.
@@ -150,21 +127,13 @@ def get_cursor_iterator(
 
     while True:
         current_page = callback(
-            limit=effective_chunk
-            if not initial_limit
-            else _min_for_limit_param(initial_limit - fetched_items, effective_chunk),
+            limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
             cursor=cursor,
         )
         yield from current_page.items
 
         fetched_items += getattr(current_page, 'count', len(current_page.items))
-
-        if isinstance(current_page, ListOfKeys):
-            cursor = current_page.next_exclusive_start_key
-        elif isinstance(current_page, ListOfRequests):
-            cursor = current_page.next_cursor
-        else:
-            raise TypeError('Unsupported page type returned by callback; expected ListOfKeys or ListOfRequests.')
+        cursor = _next_cursor_of(current_page)
 
         if not current_page.items or cursor is None or (initial_limit and fetched_items >= initial_limit):
             break
@@ -202,22 +171,37 @@ async def get_cursor_iterator_async(
 
     while True:
         current_page = await callback(
-            limit=effective_chunk
-            if not initial_limit
-            else _min_for_limit_param(initial_limit - fetched_items, effective_chunk),
+            limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
             cursor=cursor,
         )
         for item in current_page.items:
             yield item
 
         fetched_items += getattr(current_page, 'count', len(current_page.items))
-
-        if isinstance(current_page, ListOfKeys):
-            cursor = current_page.next_exclusive_start_key
-        elif isinstance(current_page, ListOfRequests):
-            cursor = current_page.next_cursor
-        else:
-            raise TypeError('Unsupported page type returned by callback; expected ListOfKeys or ListOfRequests.')
+        cursor = _next_cursor_of(current_page)
 
         if not current_page.items or cursor is None or (initial_limit and fetched_items >= initial_limit):
             break
+
+
+def _next_page_limit(initial_limit: int, fetched_items: int, effective_chunk: int) -> int:
+    """Compute the `limit` value for the next API call.
+
+    `0` means no limit on the wire (matches the Apify API contract). When both an overall `initial_limit` and a per-page
+    `effective_chunk` are set, the call is clamped to whichever is smaller; if either is unset (`0`), the other wins.
+    """
+    if not initial_limit:
+        return effective_chunk
+    remaining = initial_limit - fetched_items
+    if not effective_chunk:
+        return remaining
+    return min(remaining, effective_chunk)
+
+
+def _next_cursor_of(page: ListOfKeys | ListOfRequests) -> str | None:
+    """Return the cursor value to use for the next page of a cursor-paginated response."""
+    if isinstance(page, ListOfKeys):
+        return page.next_exclusive_start_key
+    if isinstance(page, ListOfRequests):
+        return page.next_cursor
+    raise TypeError('Unsupported page type returned by callback; expected ListOfKeys or ListOfRequests.')

src/apify_client/_resource_clients/actor_version_collection.py

@@ -49,8 +49,6 @@ def __init__(
     def list(self, *, timeout: Timeout = 'short') -> ListOfVersions:
         """List the available Actor versions.
 
-        The returned page also supports iteration: `for item in client.list()` yields individual versions.
-
         https://docs.apify.com/api/v2#/reference/actors/version-collection/get-list-of-versions
 
         Args:
@@ -145,8 +143,6 @@ def __init__(
     async def list(self, *, timeout: Timeout = 'short') -> ListOfVersions:
         """List the available Actor versions.
 
-        The returned page also supports iteration: `async for item in client.list()` yields individual versions.
-
         https://docs.apify.com/api/v2#/reference/actors/version-collection/get-list-of-versions
 
         Args:

src/apify_client/_resource_clients/request_queue.py

@@ -561,7 +561,7 @@ def iterate_requests(
 
         Simple `list_requests` does only one API call, possibly not listing all items matching the criteria.
         This method returns an iterator that is capable of making multiple API calls to retrieve all items
-        matching the criteria using the opaque ``cursor`` returned by the API.
+        matching the criteria using the opaque `cursor` returned by the API.
 
         https://docs.apify.com/api/v2#/reference/request-queues/request-collection/list-requests
 
@@ -583,7 +583,7 @@ def _callback(*, cursor: str | None = None, limit: int | None = None) -> ListOfR
             _callback,
             cursor=cursor,
             limit=limit,
-            chunk_size=chunk_size,
+            chunk_size=chunk_size or 1000,
         )
 
     def unlock_requests(self: RequestQueueClient, *, timeout: Timeout = 'long') -> UnlockRequestsResult:
@@ -1173,7 +1173,7 @@ def iterate_requests(
 
         Simple `list_requests` does only one API call, possibly not listing all items matching the criteria.
         This method returns an iterator that is capable of making multiple API calls to retrieve all items
-        matching the criteria using the opaque ``cursor`` returned by the API.
+        matching the criteria using the opaque `cursor` returned by the API.
 
         https://docs.apify.com/api/v2#/reference/request-queues/request-collection/list-requests
 
@@ -1195,7 +1195,7 @@ async def _callback(*, cursor: str | None = None, limit: int | None = None) -> L
             _callback,
             cursor=cursor,
             limit=limit,
-            chunk_size=chunk_size,
+            chunk_size=chunk_size or 1000,
         )
 
     async def unlock_requests(

tests/unit/test_client_pagination.py

@@ -88,10 +88,12 @@
 )
 
 ID_PLACEHOLDER = 'some-id'
+NORMAL_ITEMS = 2500
+EXTRA_ITEMS_UNNAMED = 100
+MAX_ITEMS_PER_PAGE = 1000
 
-
-# Inner list models whose `items: list[<specific schema>]` is relaxed to `list[dict]`.
-# Point of these tests is pagination mechanism, not internal object validation.
+# Inner list models whose `items: list[<specific schema>]` is relaxed to `list[dict]`. Point of these tests is
+# pagination mechanism, not internal object validation.
 _RELAXED_LIST_MODELS = (
     'ListOfActors',
     'ListOfBuilds',
@@ -110,9 +112,9 @@
     'ListOfWebhooks',
 )
 
-# Outer wrappers that embed a relaxed list model via `.data`. Their compiled schema pins the
-# inner's schema at construction time, so they need a forced rebuild to pick up the relaxation.
-# The wrappers themselves are not mutated — their own field annotations stay as-is.
+# Outer wrappers that embed a relaxed list model via `.data`. Their compiled schema pins the inner's schema at
+# construction time, so they need a forced rebuild to pick up the relaxation. The wrappers themselves are not mutated —
+# their own field annotations stay as-is.
 _REBUILT_RESPONSE_WRAPPERS = (
     'ListOfActorsInStoreResponse',
     'ListOfActorsResponse',
@@ -135,10 +137,10 @@
 def _relax_item_validation() -> Any:
     """Relax only the element type of `items` on paginated list models for the test run.
 
-    Pagination tests feed synthetic `{'id': N}` items that don't satisfy the real API schemas
-    (`ActorShort`, `BuildShort`, `Request`, `EnvVar`, …). Instead of bypassing validation
-    wholesale, each inner `ListOf*` model has its `items` field swapped to `list[dict]`
-    and rebuilt. Outer `.data` wrapping and every pagination-metadata field remain validated.
+    Pagination tests feed synthetic `{'id': N}` items that don't satisfy the real API schemas (`ActorShort`,
+    `BuildShort`, `Request`, `EnvVar`, …). Instead of bypassing validation wholesale, each inner `ListOf*` model has its
+    `items` field swapped to `list[dict]` and rebuilt. Outer `.data` wrapping and every pagination-metadata field remain
+    validated.
     """
     relaxed_field = FieldInfo.from_annotation(list[dict])
     originals: dict[type[BaseModel], FieldInfo] = {}
@@ -168,11 +170,6 @@ def create_items(start: int, end: int, step: int | None = None) -> list[dict[str
     return [{'id': i} for i in range(start, end, step)]
 
 
-NORMAL_ITEMS = 2500
-EXTRA_ITEMS_UNNAMED = 100
-MAX_ITEMS_PER_PAGE = 1000
-
-
 def _is_true(value: str | None) -> bool:
     """Match the `'true'` wire form produced by the client's bool→string serialization."""
     return value == 'true'
@@ -185,10 +182,9 @@ def _parse_int_param(value: str | None) -> int:
 def _handle_offset_pagination(request: Request) -> Response:
     """Serve an offset-paginated Apify API response.
 
-    The simulated platform holds 2500 items normally and an additional 100 when
-    ``unnamed=true`` is requested. Pages are capped at 1000 items regardless of the requested
-    limit, mirroring the real API. The dataset items endpoint returns items as a raw list;
-    all other endpoints wrap them in ``{'data': {...}}``.
+    The simulated platform holds 2500 items normally and an additional 100 when `unnamed=true` is requested. Pages are
+    capped at 1000 items regardless of the requested limit, mirroring the real API. The dataset items endpoint returns
+    items as a raw list; all other endpoints wrap them in `{'data': {...}}`.
     """
     params = request.args
 
@@ -238,10 +234,9 @@ def _handle_offset_pagination(request: Request) -> Response:
 def _handle_cursor_pagination(request: Request) -> Response:
     """Serve a cursor-paginated Apify API response for KVS keys and RQ requests.
 
-    Holds 2500 synthetic items whose integer `id` equals their position. Each page is capped
-    at 1000 items. KVS uses `exclusiveStartKey`; RQ accepts either the deprecated
-    `exclusiveStartId` on the initial call or the opaque `cursor` on subsequent calls. All
-    three values encode the last-seen item id as a string — the next page starts at id + 1.
+    Holds 2500 synthetic items whose integer `id` equals their position. Each page is capped at 1000 items. KVS uses
+    `exclusiveStartKey`; RQ accepts either the deprecated `exclusiveStartId` on the initial call or the opaque `cursor`
+    on subsequent calls. All three values encode the last-seen item id as a string — the next page starts at id + 1.
     """
     params = request.args
     limit = _parse_int_param(params.get('limit'))
@@ -303,9 +298,8 @@ def _make_async_client(httpserver: HTTPServer) -> ApifyClientAsync:
     return ApifyClientAsync(token='test', api_url=httpserver.url_for('/'))
 
 
-# Map resource-client class name to a factory that, given an `ApifyClient`/`ApifyClientAsync`,
-# returns the sub-client under test. Usable for both sync and async since every accessor is
-# available symmetrically on both root clients.
+# Map resource-client class name to a factory that, given an `ApifyClient`/`ApifyClientAsync`, returns the sub-client
+# under test. Usable for both sync and async since every accessor is available symmetrically on both root clients.
 _CLIENT_FACTORIES: dict[str, Callable[[Any], Any]] = {
     'ActorCollectionClient': lambda c: c.actors(),
     'ScheduleCollectionClient': lambda c: c.schedules(),
@@ -390,8 +384,8 @@ def __hash__(self) -> int:
 
 TEST_CASES = (
     _PaginationCase('No options normal', {}, create_items(0, 2500), OPTIONS_CLIENTS),
-    # These clients can't iterate over all items if there is more of them than the API limit as they offer no
-    # pagination parameters.
+    # These clients can't iterate over all items if there is more of them than the API limit as they offer no pagination
+    # parameters.
     _PaginationCase('No options limited', {}, create_items(0, 1000), NO_OPTIONS_CLIENTS),
     _PaginationCase('Limit', {'limit': 1100}, create_items(0, 1100), OPTIONS_CLIENTS),
     _PaginationCase('Out of range limit', {'limit': 3000}, create_items(0, 2500), OPTIONS_CLIENTS),
@@ -470,9 +464,8 @@ def __hash__(self) -> int:
 def _generate_test_params(client_set: Literal['collection', 'dataset', 'kvs', 'rq']) -> list[ParameterSet]:
     """Build the pytest parameter set for the given client category.
 
-    Each parameter carries the resource-client class name; the test body instantiates
-    the real client against the `httpserver` URL and looks up the factory in
-    `_CLIENT_FACTORIES`.
+    Each parameter carries the resource-client class name; the test body instantiates the real client against the
+    `httpserver` URL and looks up the factory in `_CLIENT_FACTORIES`.
     """
     client_names = _CLIENT_SET_NAMES[client_set]
     return [