chore(airflow): Add 3.1.5, remove 2.10.5 and 3.0.1 (#1405)

* chore(airflow): Remove unused constraints for 2.10.5

* chore(airflow): Remove unused constraints for 3.0.1

* chore(airflow): Remove unused patchable config for 2.10.5

* chore(airflow): Remove unused patchable config file for 3.0.1

* chore(airflow): Remove 2.10.5 and 3.0.1

* chore(airflow): Add 3.1.5

It should be noted that Cython had to be constrained to >= 3.1.0
which means no specific version is used. This is due to conflicts
when gssapi and krb5 are built. gssapi (1.10.1)  requires 3.1.3,
but krb5 (0.9.0) requires 3.2.1.

It might additionally also be possible to use Python 3.13 and a
newer uv version, as Airflow supports Python 3.13 and uses uv
0.9.x. This will we explored in a future commit.

The cyclonedx-bom generator was bumped to the latest version, but
that now requires an if statement in the Dockerfile. It might be
possible to also use the newer version for Airflow 3.0.6.

Author: Techassi

airflow/Dockerfile

@@ -168,7 +168,11 @@ uv pip install --no-cache-dir /tmp/opa_auth_manager-0.1.0-py3-none-any.whl
 
 # Create the SBOM for Airflow
 # Important: All `pip install` commands must be above this line, otherwise the SBOM will be incomplete
-cyclonedx-py environment --schema-version 1.5 --outfile /tmp/sbom.json
+if [ "$PRODUCT_VERSION" == "3.1.5" ]; then
+  cyclonedx-py environment --spec-version 1.5 --output-file /tmp/sbom.json
+else
+  cyclonedx-py environment --schema-version 1.5 --outfile /tmp/sbom.json
+fi
 uv pip uninstall cyclonedx-bom
 
 # Break circular dependencies by removing the apache-airflow dependency from the providers

airflow/boil-config.toml

@@ -1,3 +1,4 @@
+# Deprecated since SDP 25.11
 [versions."2.9.3".local-images]
 "shared/statsd-exporter" = "0.28.0"
 vector = "0.52.0"
@@ -14,69 +15,76 @@ airflow-extras-other = "async,amazon,celery,cncf.kubernetes,docker,dask,elastics
 opa-auth-manager = "airflow-2"
 nodejs-version = "20"
 
-[versions."2.10.5".local-images]
+# LTS
+[versions."3.0.6".local-images]
 "shared/statsd-exporter" = "0.28.0"
 vector = "0.52.0"
 stackable-devel = "1.0.0"
 
-[versions."2.10.5".build-arguments]
+[versions."3.0.6".build-arguments]
 python-version = "3.12"
 git-sync-version = "v4.4.1"
 s3fs-version = "2024.9.0"
 cyclonedx-bom-version = "6.0.0"
 tini-version = "0.19.0"
 uv-version = "0.7.8"
-airflow-extras-other = "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino"
-opa-auth-manager = "airflow-2"
-nodejs-version = "20"
 
-[versions."3.0.1".local-images]
-"shared/statsd-exporter" = "0.28.0"
-vector = "0.52.0"
-stackable-devel = "1.0.0"
+# Airflow extras are defined in separate lists to make them easier to check against the links below. The lists will be concatenated and duplicates removed in the dockerfile.
+# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#core-airflow-extras
+airflow-extras-core = "async,graphviz,kerberos,otel,sentry,standard,statsd"
+
+# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#meta-airflow-package-extras
+airflow-extras-meta = "aiobotocore,cloudpickle,github-enterprise,google-auth,graphviz,ldap,leveldb,pandas,polars,rabbitmq,s3fs,saml,uv"
+
+# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#apache-software-extras
+airflow-extras-provider-apache = "apache-beam,apache-cassandra,apache-drill,apache-druid,apache-flink,apache-hdfs,apache-hive,apache-iceberg,apache-impala,apache-kafka,apache-kylin,apache-livy,apache-pig,apache-pinot"
+
+# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#external-services-extras
+airflow-extras-external-services = "airbyte,alibaba,apprise,amazon,asana,atlassian-jira,microsoft-azure,cloudant,cohere,databricks,datadog,dbt-cloud,dingding,discord,facebook,github,google,hashicorp,openai,opsgenie,pagerduty,pgvector,pinecone,qdrant,salesforce,sendgrid,segment,slack,snowflake,tableau,tabular,telegram,vertica,weaviate,yandex,ydb,zendesk"
+
+# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#locally-installed-software-extras
+airflow-extras-locally-installed-software = "arangodb,celery,cncf-kubernetes,docker,edge3,elasticsearch,exasol,fab,git,github,influxdb,jenkins,mongo,microsoft-mssql,neo4j,odbc,openfaas,oracle,postgres,presto,redis,samba,singularity,teradata,trino"
+
+# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#other-extras
+airflow-extras-other = "common-compat,common-io,common-messaging,common-sql,ftp,grpc,http,imap,jdbc,microsoft-psrp,microsoft-winrm,openlineage,opensearch,papermill,sftp,smtp,sqlite,ssh"
 
-[versions."3.0.1".build-arguments]
-python-version = "3.12"
-git-sync-version = "v4.4.1"
-s3fs-version = "2024.9.0"
-cyclonedx-bom-version = "6.0.0"
-tini-version = "0.19.0"
-uv-version = "0.7.8"
-airflow-extras-other = "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino"
 opa-auth-manager = "airflow-3"
 nodejs-version = "20"
 
-[versions."3.0.6".local-images]
+# Supported
+[versions."3.1.5".local-images]
 "shared/statsd-exporter" = "0.28.0"
 vector = "0.52.0"
 stackable-devel = "1.0.0"
 
-[versions."3.0.6".build-arguments]
+[versions."3.1.5".build-arguments]
+# NOTE (@Techassi): Maybe this version can be bumped
 python-version = "3.12"
-git-sync-version = "v4.4.1"
-s3fs-version = "2024.9.0"
-cyclonedx-bom-version = "6.0.0"
+git-sync-version = "v4.5.1"
+s3fs-version = "2026.1.0"
+cyclonedx-bom-version = "7.2.1"
 tini-version = "0.19.0"
-uv-version = "0.7.8"
+# NOTE (@Techassi): Maybe this version can be bumped
+uv-version = "0.7.22"
 
 # Airflow extras are defined in separate lists to make them easier to check against the links below. The lists will be concatenated and duplicates removed in the dockerfile.
-# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#core-airflow-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#core-airflow-extras
 airflow-extras-core = "async,graphviz,kerberos,otel,sentry,standard,statsd"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#meta-airflow-package-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#meta-airflow-package-extras
 airflow-extras-meta = "aiobotocore,cloudpickle,github-enterprise,google-auth,graphviz,ldap,leveldb,pandas,polars,rabbitmq,s3fs,saml,uv"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#apache-software-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#apache-software-extras
 airflow-extras-provider-apache = "apache-beam,apache-cassandra,apache-drill,apache-druid,apache-flink,apache-hdfs,apache-hive,apache-iceberg,apache-impala,apache-kafka,apache-kylin,apache-livy,apache-pig,apache-pinot"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#external-services-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#external-services-extras
 airflow-extras-external-services = "airbyte,alibaba,apprise,amazon,asana,atlassian-jira,microsoft-azure,cloudant,cohere,databricks,datadog,dbt-cloud,dingding,discord,facebook,github,google,hashicorp,openai,opsgenie,pagerduty,pgvector,pinecone,qdrant,salesforce,sendgrid,segment,slack,snowflake,tableau,tabular,telegram,vertica,weaviate,yandex,ydb,zendesk"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#locally-installed-software-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#locally-installed-software-extras
 airflow-extras-locally-installed-software = "arangodb,celery,cncf-kubernetes,docker,edge3,elasticsearch,exasol,fab,git,github,influxdb,jenkins,mongo,microsoft-mssql,neo4j,odbc,openfaas,oracle,postgres,presto,redis,samba,singularity,teradata,trino"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#other-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.1.5/extra-packages-ref.html#other-extras
 airflow-extras-other = "common-compat,common-io,common-messaging,common-sql,ftp,grpc,http,imap,jdbc,microsoft-psrp,microsoft-winrm,openlineage,opensearch,papermill,sftp,smtp,sqlite,ssh"
 
 opa-auth-manager = "airflow-3"
-nodejs-version = "20"
+nodejs-version = "22"