Skip to content

[PULP-1082] Add data-requires-python to Simple HTML API #1065

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jobselko wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[PULP-1082] Add data-requires-python to Simple HTML API #1065

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES/1054.feature
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added data-requires-python to Simple HTML API.
5 changes: 3 additions & 2 deletions pulp_python/app/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
</html>
"""

# TODO in the future: data-requires-python (PEP 503)
# TODO in the future: data-yanked (not implemented yet because it is mutable)
simple_detail_template = """<!DOCTYPE html>
<html>
<head>
Expand All @@ -58,6 +58,7 @@
<h1>Links for {{ project_name }}</h1>
{%- for pkg in project_packages %}
<a href="{{ pkg.url }}#sha256={{ pkg.sha256 }}" rel="internal"
{%- if pkg.requires_python %} data-requires-python="{{ pkg.requires_python }}" {%- endif %}
{%- if pkg.metadata_sha256 %} data-dist-info-metadata="sha256={{ pkg.metadata_sha256 }}" data-core-metadata="sha256={{ pkg.metadata_sha256 }}"
{%- endif %} {% if pkg.provenance -%}
data-provenance="{{ pkg.provenance }}"{% endif %}>{{ pkg.filename }}</a><br/>
Expand Down Expand Up @@ -501,7 +502,7 @@ def write_simple_index(project_names, streamed=False):

def write_simple_detail(project_name, project_packages, streamed=False):
"""Writes the simple detail page of a package."""
detail = Template(simple_detail_template)
detail = Template(simple_detail_template, autoescape=True)
context = {
"SIMPLE_API_VERSION": SIMPLE_API_VERSION,
"project_name": project_name,
Expand Down
71 changes: 38 additions & 33 deletions pulp_python/tests/functional/api/test_pypi_simple_api.py
View file
Open in desktop
Copy link
Contributor Author

@jobselko jobselko Jan 14, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I rewrote tests because shelf-reader does not contain requires-python (and twine 5.1.0 contains also data-yanked if we will ever need it)

Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,23 @@

from pulp_python.tests.functional.constants import (
PYPI_SERIAL_CONSTANT,
PYTHON_EGG_FILENAME,
PYTHON_EGG_SHA256,
PYTHON_EGG_URL,
PYTHON_SM_FIXTURE_CHECKSUMS,
PYTHON_SM_FIXTURE_RELEASES,
PYTHON_SM_PROJECT_SPECIFIER,
PYTHON_WHEEL_FILENAME,
PYTHON_WHEEL_METADATA_SHA256,
PYTHON_WHEEL_SHA256,
PYTHON_WHEEL_URL,
PYTHON_XS_FIXTURE_CHECKSUMS,
TWINE_EGG_FILENAME,
TWINE_EGG_REQUIRES_PYTHON,
TWINE_EGG_SHA256,
TWINE_EGG_SIZE,
TWINE_EGG_URL,
TWINE_FIXTURE_CHECKSUMS,
TWINE_FIXTURE_METADATA_SHA256,
TWINE_FIXTURE_REQUIRES_PYTHON,
TWINE_WHEEL_FILENAME,
TWINE_WHEEL_METADATA_SHA256,
TWINE_WHEEL_REQUIRES_PYTHON,
TWINE_WHEEL_SHA256,
TWINE_WHEEL_SIZE,
TWINE_WHEEL_URL,
)
from pulp_python.tests.functional.utils import ensure_simple

Expand Down Expand Up @@ -55,30 +61,27 @@ def test_simple_html_detail_api(
python_distribution_factory,
python_repo_factory,
):
content_1 = python_content_factory(PYTHON_WHEEL_FILENAME, url=PYTHON_WHEEL_URL)
content_2 = python_content_factory(PYTHON_EGG_FILENAME, url=PYTHON_EGG_URL)
content_1 = python_content_factory(TWINE_WHEEL_FILENAME, url=TWINE_WHEEL_URL)
content_2 = python_content_factory(TWINE_EGG_FILENAME, url=TWINE_EGG_URL)
body = {"add_content_units": [content_1.pulp_href, content_2.pulp_href]}

repo = python_repo_factory()
monitor_task(python_bindings.RepositoriesPythonApi.modify(repo.pulp_href, body).task)
distro = python_distribution_factory(repository=repo)

url = f'{urljoin(distro.base_url, "simple/")}shelf-reader'
url = f'{urljoin(distro.base_url, "simple/")}twine'
headers = {"Accept": PYPI_SIMPLE_V1_HTML}

response = requests.get(url, headers=headers)
assert response.headers["Content-Type"] == PYPI_SIMPLE_V1_HTML
assert response.headers["X-PyPI-Last-Serial"] == str(PYPI_SERIAL_CONSTANT)

metadata_sha_digests = {
PYTHON_WHEEL_FILENAME: PYTHON_WHEEL_METADATA_SHA256,
PYTHON_EGG_FILENAME: None, # egg files should not have metadata
}
proper, msgs = ensure_simple(
urljoin(distro.base_url, "simple/"),
{"shelf-reader": [PYTHON_WHEEL_FILENAME, PYTHON_EGG_FILENAME]},
sha_digests=PYTHON_XS_FIXTURE_CHECKSUMS,
metadata_sha_digests=metadata_sha_digests,
{"twine": [TWINE_WHEEL_FILENAME, TWINE_EGG_FILENAME]},
sha_digests=TWINE_FIXTURE_CHECKSUMS,
metadata_sha_digests=TWINE_FIXTURE_METADATA_SHA256,
requires_python=TWINE_FIXTURE_REQUIRES_PYTHON,
)
assert proper, f"Simple API validation failed: {msgs}"

Expand Down Expand Up @@ -114,15 +117,15 @@ def test_simple_json_detail_api(
python_distribution_factory,
python_repo_factory,
):
content_1 = python_content_factory(PYTHON_WHEEL_FILENAME, url=PYTHON_WHEEL_URL)
content_2 = python_content_factory(PYTHON_EGG_FILENAME, url=PYTHON_EGG_URL)
content_1 = python_content_factory(TWINE_WHEEL_FILENAME, url=TWINE_WHEEL_URL)
content_2 = python_content_factory(TWINE_EGG_FILENAME, url=TWINE_EGG_URL)
body = {"add_content_units": [content_1.pulp_href, content_2.pulp_href]}

repo = python_repo_factory()
monitor_task(python_bindings.RepositoriesPythonApi.modify(repo.pulp_href, body).task)
distro = python_distribution_factory(repository=repo)

url = f'{urljoin(distro.base_url, "simple/")}shelf-reader'
url = f'{urljoin(distro.base_url, "simple/")}twine'
headers = {"Accept": PYPI_SIMPLE_V1_JSON}

response = requests.get(url, headers=headers)
Expand All @@ -131,29 +134,31 @@ def test_simple_json_detail_api(

data = response.json()
assert data["meta"] == {"api-version": API_VERSION, "_last-serial": PYPI_SERIAL_CONSTANT}
assert data["name"] == "shelf-reader"
assert data["name"] == "twine"
assert data["files"]
assert data["versions"] == ["0.1"]
assert data["versions"] == ["5.1.0"]

# Check data of a wheel
file_whl = next((i for i in data["files"] if i["filename"] == PYTHON_WHEEL_FILENAME), None)
file_whl = next((i for i in data["files"] if i["filename"] == TWINE_WHEEL_FILENAME), None)
assert file_whl is not None, "wheel file not found"
assert file_whl["url"]
assert file_whl["hashes"] == {"sha256": PYTHON_WHEEL_SHA256}
assert file_whl["requires-python"] is None
assert file_whl["data-dist-info-metadata"] == {"sha256": PYTHON_WHEEL_METADATA_SHA256}
assert file_whl["core-metadata"] == {"sha256": PYTHON_WHEEL_METADATA_SHA256}
assert file_whl["size"] == 22455
assert file_whl["hashes"] == {"sha256": TWINE_WHEEL_SHA256}
assert file_whl["requires-python"] == TWINE_WHEEL_REQUIRES_PYTHON
assert file_whl["data-dist-info-metadata"] == {"sha256": TWINE_WHEEL_METADATA_SHA256}
assert file_whl["core-metadata"] == {"sha256": TWINE_WHEEL_METADATA_SHA256}
assert file_whl["size"] == TWINE_WHEEL_SIZE
assert file_whl["upload-time"] is not None
assert file_whl["provenance"] is None

# Check data of a tarball
file_tar = next((i for i in data["files"] if i["filename"] == PYTHON_EGG_FILENAME), None)
file_tar = next((i for i in data["files"] if i["filename"] == TWINE_EGG_FILENAME), None)
assert file_tar is not None, "tar file not found"
assert file_tar["url"]
assert file_tar["hashes"] == {"sha256": PYTHON_EGG_SHA256}
assert file_tar["requires-python"] is None
assert file_tar["hashes"] == {"sha256": TWINE_EGG_SHA256}
assert file_tar["requires-python"] == TWINE_EGG_REQUIRES_PYTHON
assert file_tar["data-dist-info-metadata"] is False
assert file_tar["core-metadata"] is False
assert file_tar["size"] == 19097
assert file_tar["size"] == TWINE_EGG_SIZE
assert file_tar["upload-time"] is not None
assert file_tar["provenance"] is None

Expand Down
26 changes: 26 additions & 0 deletions pulp_python/tests/functional/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,32 @@
# maybe add description, license is long for this one
}

# twine pkg data for PyPI Simple API
TWINE_EGG_FILENAME = "twine-5.1.0.tar.gz"
TWINE_EGG_URL = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), TWINE_EGG_FILENAME)
TWINE_EGG_SHA256 = "4d74770c88c4fcaf8134d2a6a9d863e40f08255ff7d8e2acb3cbbd57d25f6e9d"
TWINE_EGG_SIZE = 224997
TWINE_EGG_REQUIRES_PYTHON = ">=3.8"

TWINE_WHEEL_FILENAME = "twine-5.1.0-py3-none-any.whl"
TWINE_WHEEL_URL = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), TWINE_WHEEL_FILENAME)
TWINE_WHEEL_SHA256 = "fe1d814395bfe50cfbe27783cb74efe93abeac3f66deaeb6c8390e4e92bacb43"
TWINE_WHEEL_SIZE = 38563
TWINE_WHEEL_REQUIRES_PYTHON = ">=3.8"
TWINE_WHEEL_METADATA_SHA256 = "0ac5cf457bd47512b3477949ff6274cc2258414f3e1f136e049585aac92e4ddb"

TWINE_FIXTURE_CHECKSUMS = {
TWINE_EGG_FILENAME: TWINE_EGG_SHA256,
TWINE_WHEEL_FILENAME: TWINE_WHEEL_SHA256,
}
TWINE_FIXTURE_METADATA_SHA256 = {
TWINE_WHEEL_FILENAME: TWINE_WHEEL_METADATA_SHA256,
TWINE_EGG_FILENAME: None, # egg files should not have metadata
}
TWINE_FIXTURE_REQUIRES_PYTHON = {
TWINE_WHEEL_FILENAME: TWINE_EGG_REQUIRES_PYTHON,
TWINE_EGG_FILENAME: TWINE_WHEEL_REQUIRES_PYTHON,
}

# Current tests use PYTHON_FIXTURES_URL with an 'S', remove after adding api tests
PYTHON_FIXTURE_URL = urljoin(PULP_FIXTURES_BASE_URL, "python-pypi/")
Expand Down
39 changes: 37 additions & 2 deletions pulp_python/tests/functional/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,36 @@ def _validate_metadata_sha_digest(link, filename, metadata_sha_digests):
return msgs


def ensure_simple(simple_url, packages, sha_digests=None, metadata_sha_digests=None):
def _validate_requires_python(link, filename, requires_python, page_content):
"""
Validate data-requires-python attribute for a release link.
"""
expected_requires_python = requires_python.get(filename) if requires_python else None
attr_value = link.get("data-requires-python")

msgs = ""
if attr_value != expected_requires_python:
if expected_requires_python:
msgs += (
f"\nFile {filename} has incorrect data-requires-python: "
f"expected '{expected_requires_python}', got '{attr_value}'"
)
else:
msgs += f"\nFile {filename} should not have data-requires-python but has '{attr_value}'"

# Check HTML escaping
if expected_requires_python and any(char in expected_requires_python for char in [">", "<"]):
escaped_value = expected_requires_python.replace(">", "&gt;").replace("<", "&lt;")
escaped_attr = f'data-requires-python="{escaped_value}"'
if escaped_attr not in page_content:
msgs += f"\nFile {filename} has unescaped < or > in data-requires-python attribute"

return msgs


def ensure_simple(
simple_url, packages, sha_digests=None, metadata_sha_digests=None, requires_python=None
):
"""
Tests that the simple api at `url` matches the packages supplied.
`packages`: dictionary of form {package_name: [release_filenames]}
Expand All @@ -40,7 +69,8 @@ def ensure_simple(simple_url, packages, sha_digests=None, metadata_sha_digests=N

def explore_links(page_url, page_name, links_found, msgs):
legit_found_links = []
page = html.fromstring(requests.get(page_url).text)
page_content = requests.get(page_url).text
page = html.fromstring(page_content)
page_links = page.xpath("/html/body/a")
for link in page_links:
if link.text in links_found:
Expand All @@ -52,6 +82,11 @@ def explore_links(page_url, page_name, links_found, msgs):
# Check metadata SHA digest if provided
if metadata_sha_digests and page_name == "release":
msgs += _validate_metadata_sha_digest(link, link.text, metadata_sha_digests)
# Check requires-python if provided
if requires_python and page_name == "release":
msgs += _validate_requires_python(
link, link.text, requires_python, page_content
)
else:
msgs += f"\nFound {page_name} link without href {link.text}"
else:
Expand Down