PYTHON-5047 Improve testing of publish workflows and fix integration tests (#943)

* PYTHON-5047 Improve testing of publish workflows

* fix handling of testpypi

* fix dry run logic

* fix dry run logic

* fix integ test

* debug

* fix integ test

* update crypt_shared_dir

* test

* update path

* update test

* use upstream

* use upstream

* use fork

* Get drivers-evergreen-tools from master again

---------

Co-authored-by: Casey Clements <caseyclements@users.noreply.github.com>

Author: blink1073

.github/workflows/release-python.yml

@@ -8,18 +8,25 @@ on:
         required: true
       following_version:
         description: "The post (dev) version to set"
-        required: true
       dry_run:
         description: "Dry Run?"
         default: false
         type: boolean
+  schedule:
+    - cron:  '30 5 * * *'
 
 env:
   # Changes per repo
   PRODUCT_NAME: PyMongoCrypt
   # Changes per branch
   SILK_ASSET_GROUP: pymongocrypt
   EVERGREEN_PROJECT: libmongocrypt
+  # Constant
+  # inputs will be empty on a scheduled run.  so, we only set dry_run
+  # to 'false' when the input is set to 'false'.
+  DRY_RUN: ${{ ! contains(inputs.dry_run, 'false') }}
+  FOLLOWING_VERSION: ${{ inputs.following_version || '' }}
+  VERSION: ${{ inputs.version || '10.10.10.10' }}
 
 defaults:
   run:
@@ -48,9 +55,9 @@ jobs:
       - uses: mongodb-labs/drivers-github-tools/python/pre-publish@v2
         id: pre-publish
         with:
-          version: ${{ inputs.version }}
+          version: ${{ env.VERSION }}
           working_directory: ./bindings/python
-          dry_run: ${{ inputs.dry_run }}
+          dry_run: ${{ env.DRY_RUN }}
           tag_template: "pymongocrypt-${VERSION}"
           tag_message_template:  "Release PyMongoCrypt ${VERSION}"
 
@@ -80,8 +87,14 @@ jobs:
         with:
           name: all-dist-${{ github.run_id }}
           path: dist/
+      - name: Publish package distributions to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
+          attestations: ${{ env.DRY_RUN }}
       - name: Publish package distributions to PyPI
-        if: startsWith(inputs.dry_run, 'false')
+        if: startsWith(env.DRY_RUN, 'false')
         uses: pypa/gh-action-pypi-publish@release/v1
 
   post-publish:
@@ -106,12 +119,12 @@ jobs:
           artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }}
       - uses: mongodb-labs/drivers-github-tools/python/post-publish@v2
         with:
-          version: ${{ inputs.version }}
-          following_version: ${{ inputs.following_version }}
+          version: ${{ env.VERSION }}
+          following_version: ${{ env.FOLLOWING_VERSION }}
           working_directory: ./bindings/python
           product_name: ${{ env.PRODUCT_NAME }}
           silk_asset_group: ${{ env.SILK_ASSET_GROUP }}
           evergreen_project: ${{ env.EVERGREEN_PROJECT }}
           tag_template: "pymongocrypt-${VERSION}"
           token: ${{ github.token }}
-          dry_run: ${{ inputs.dry_run }}
+          dry_run: ${{ env.DRY_RUN }}

bindings/python/.evergreen/integ-setup.sh

@@ -38,6 +38,8 @@ EOT
 
 # Get the secrets
 bash $DRIVERS_TOOLS/.evergreen/csfle/setup-secrets.sh
+# Start the csfle servers
+bash $DRIVERS_TOOLS/.evergreen/csfle/start-servers.sh
 
 # Clone mongo-python-driver
 git clone https://github.com/mongodb/mongo-python-driver.git ${PYMONGO_DIR}

bindings/python/.evergreen/integ-test.sh

@@ -11,6 +11,7 @@ BASE_PYTHON=$(find_python3)
 # MONGOCRYPT_DIR is set by libmongocrypt/.evergreen/config.yml
 MONGOCRYPT_DIR="$MONGOCRYPT_DIR"
 CRYPT_SHARED_DIR="$DRIVERS_TOOLS"
+MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
 
 MACHINE=$(uname -m)
 if [ $MACHINE == "aarch64" ]; then
@@ -33,11 +34,15 @@ pushd $PYMONGO_DIR
 pip install -e ".[test,encryption]"
 source ${DRIVERS_TOOLS}/.evergreen/csfle/secrets-export.sh
 set -x
-TEST_CRYPT_SHARED=1 DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:${DYLD_FALLBACK_LIBRARY_PATH:-} \
-    LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:${LD_LIBRARY_PATH-} \
-    PATH=$CRYPT_SHARED_DIR:$PATH \
-    AUTH=auth SSL=ssl \
-    .evergreen/run-tests.sh -m encryption
+export DB_USER="bob"
+export DB_PASSWORD="pwd123"
+export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
+export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
+export DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:${DYLD_FALLBACK_LIBRARY_PATH:-}
+export LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:${LD_LIBRARY_PATH-}
+export PATH=$CRYPT_SHARED_DIR:$MONGODB_BINARIES:$PATH
+export TEST_CRYPT_SHARED="1"
+pytest --maxfail=10 -v -m encryption
 
 popd
 deactivate