diff --git a/contrib/dbt_factory/.gitignore b/contrib/dbt_factory/.gitignore new file mode 100644 index 00000000..5a79248b --- /dev/null +++ b/contrib/dbt_factory/.gitignore @@ -0,0 +1,19 @@ +.venv/ +__pycache__/ +*.pyc +.databricks/ +logs/ +# Ignore installed dbt packages but keep the directory (via .gitkeep) so the +# `dbt_packages/**` sync.include in databricks.yml always matches at least one file. +dbt_packages/* +!dbt_packages/.gitkeep +dbt_profiles/.user.yml +uv.lock + +# dbt build artifacts, but keep the committed manifest that resources/__init__.py reads at +# deploy time. Regenerate it with `make manifest`. +target/* +!target/manifest.json + +# Serverless base-environment file, regenerated on every `bundle deploy` by resources/__init__.py. +dbt_serverless_env.yaml diff --git a/contrib/dbt_factory/Makefile b/contrib/dbt_factory/Makefile new file mode 100644 index 00000000..cb40b3a0 --- /dev/null +++ b/contrib/dbt_factory/Makefile @@ -0,0 +1,40 @@ +.PHONY: setup deps manifest validate deploy run test test-update-expected-tasks test-e2e + +# Install dependencies into the .venv used by the bundle (databricks.yml -> python.venv_path). +setup: + uv sync --dev + +# Install dbt package dependencies declared in packages.yml / dependencies.yml (if any). +# `dbt deps` empties dbt_packages/, so restore the .gitkeep afterwards; it keeps the directory +# present so the `dbt_packages/**` sync.include in databricks.yml always matches at least one file. +deps: + uv run dbt deps + touch dbt_packages/.gitkeep + +# Regenerate the dbt manifest that resources/__init__.py reads at deploy time. +# `dbt parse` does not connect to a warehouse; it only reads the project files. +manifest: deps + uv run dbt parse --profiles-dir dbt_profiles + +validate: + databricks bundle validate + +# Regenerate the manifest and deploy the generated job to the dev target. +deploy: manifest + databricks bundle deploy --target dev + +run: + databricks bundle run dbt_factory_job + +test: + uv run pytest tests + +# Refresh the saved "expected tasks" file the tests compare against, after you intentionally +# change the generated output. +test-update-expected-tasks: + uv run python tests/update_expected_tasks.py + +# End-to-end test: generate a project from the template, deploy the factory-generated job to your +# workspace, run it, verify the output, and tear it all down. Needs DBT_FACTORY_* env (see tests/e2e/run_e2e.py). +test-e2e: + uv run python tests/e2e/run_e2e.py diff --git a/contrib/dbt_factory/NOTICE b/contrib/dbt_factory/NOTICE new file mode 100644 index 00000000..702735ef --- /dev/null +++ b/contrib/dbt_factory/NOTICE @@ -0,0 +1,34 @@ +This project includes code adapted from the "databricks-dbt-factory" library. + + Source: https://github.com/mwojtyczka/databricks-dbt-factory + Full history: https://github.com/mwojtyczka/databricks-dbt-factory/commits/main + Adapted from: commit 29866edb82de28c33fe888be67d451068d4d7f94 (v0.3.5) + Location: src/databricks_dbt_factory/ + +The code under src/databricks_dbt_factory/ originates from that repository (trimmed to the +serverless + notebook-task subset used here, then reformatted to this repository's style) and is +provided under the MIT +license reproduced below, which this NOTICE preserves for attribution. All other files are +part of the bundle-examples repository and are covered by that repository's license. + +-------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2024-present mwojtyczka + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/contrib/dbt_factory/README.md b/contrib/dbt_factory/README.md new file mode 100644 index 00000000..45b193ac --- /dev/null +++ b/contrib/dbt_factory/README.md @@ -0,0 +1,247 @@ +# dbt_factory + +This example runs a [dbt](https://docs.getdbt.com/) project on Databricks as a +**Databricks Workflow with one task per dbt object** (model, seed, snapshot, test) instead of +running the whole project as a single opaque task. + +It does this by combining two pieces: + +* **[databricks-dbt-factory](https://github.com/mwojtyczka/databricks-dbt-factory)** — a small + library that reads a dbt `manifest.json` and expands it into Databricks job tasks, wiring up + the dependencies between them. Its source is included under `src/databricks_dbt_factory/` + (see [`NOTICE`](NOTICE) for attribution and license). +* **[PyDABs](https://docs.databricks.com/dev-tools/bundles/python)** — the Declarative Automation + Bundles Python resources hook. At `databricks bundle deploy` time the Databricks CLI calls + `load_resources` in [`resources/__init__.py`](resources/__init__.py), which runs the factory + against the manifest and returns the generated job. + +The result: **no per-model job YAML is checked in**. The task graph is generated on the fly from +the dbt manifest each time you deploy. + +## Why one task per dbt object? + +By default dbt's integration with Databricks Workflows treats the whole project as a single +task — a black box. Expanding it into one task per object gives: + +* **Faster execution** — independent models run in parallel, and the notebook task type runs dbt + from a pre-built serverless base environment, avoiding a dependency install on every task. +* **Visibility & simplified troubleshooting** — pinpoint and fix issues at the model level right + in the Databricks Workflows UI. +* **Enhanced logging & notifications** — per-task logs and precise, model-level error alerts. +* **Improved retriability** — retry only the failed model tasks without rerunning the whole project. +* **Seamless testing** — dbt data tests run as their own tasks right after each model finishes, + for faster validation and feedback. + +This example uses **serverless compute** and the **notebook task type** (each task triggers dbt +through a small runner notebook using the `dbtRunner` Python API) for the fastest task start +times. See the [databricks-dbt-factory README](https://github.com/mwojtyczka/databricks-dbt-factory#benefits) +for more. + +## How it works + +The [`dbt-factory` template](../templates/dbt-factory) scaffolds a self-contained project. +From then on, each `databricks bundle deploy` regenerates the Workflow from your current dbt +manifest — add or remove a model and the task graph follows on the next deploy, with no per-model +YAML to maintain. + +```mermaid +flowchart TD + subgraph setup["One-time setup"] + T["dbt-factory bundle template"] -->|databricks bundle init| B["Scaffolded project:
dbt project + PyDABs hook + factory code"] + X["Existing dbt project
(optional)"] -.->|move models/seeds/... into src/| B + end + subgraph deploy["Every deploy"] + C["make manifest
(dbt parse)"] --> D["target/manifest.json"] + D --> E["databricks bundle deploy"] + E --> F["PyDABs load_resources reads the
manifest and generates the job"] + end + subgraph runtime["At run time — serverless"] + G["Databricks Workflow:
one task per model / seed / snapshot / test"] --> H["Each task triggers dbt
via the runner notebook"] + H --> I[("SQL warehouse")] + end + B --> C + F --> G + + classDef optional stroke:#999,stroke-dasharray:5 4,color:#888; + class X optional; +``` + +## Project structure + +``` +dbt_factory/ +├── databricks.yml # Bundle definition; wires up the PyDABs `load_resources` hook +├── dbt_project.yml # dbt project (models under src/models, etc.) +├── dbt_profiles/profiles.yml # dbt profiles for the deployed job (dev / prod targets) +├── profile_template.yml # prompts for `dbt init` (local development) +├── resources/__init__.py # PyDABs glue: manifest -> generated job (the only integration code) +├── src/ +│ ├── models/ # your dbt models (example: orders_raw, orders_daily) +│ └── databricks_dbt_factory/ # vendored factory library (trimmed; see NOTICE) +├── target/manifest.json # committed dbt manifest, read at deploy time (regenerate with `make manifest`) +├── tests/ # tests for the vendored factory + the PyDABs integration +├── pyproject.toml # dependencies (installed into .venv via `uv sync`) +└── Makefile # convenience targets: setup, manifest, validate, deploy, run, test, test-e2e +``` + +## Setup + +1. Install the [Databricks CLI](https://docs.databricks.com/dev-tools/cli/databricks-cli.html) + and the [uv](https://docs.astral.sh/uv/) package manager. + +2. Authenticate to your Databricks workspace: + ``` + $ databricks configure + ``` + +3. Install dependencies into the `.venv` the bundle uses: + ``` + $ make setup # == uv sync --dev + ``` + +4. Edit `dbt_profiles/profiles.yml` and set your SQL warehouse `http_path`, `catalog`, and + `schema`. Set the workspace host in `databricks.yml` (and the prod `root_path` / permissions). + +## The dbt manifest + +`resources/__init__.py` reads `target/manifest.json` at deploy time to build the task graph. A +manifest is committed so the bundle deploys out of the box. **After you change your models, +regenerate it:** + +``` +$ make manifest # == uv run dbt deps && uv run dbt parse +``` + +`dbt parse` only reads your project files; it does not connect to a warehouse. The manifest +location is configurable — point at a different file via the `DBT_MANIFEST_PATH` environment +variable or by editing `MANIFEST_PATH` in `resources/__init__.py`. + +> **Faster task startup (automatic).** `make manifest` also writes `target/partial_parse.msgpack` +> next to the manifest. The bundle syncs it (`sync.include` in `databricks.yml`) and each notebook +> task injects it to **skip dbt's parse phase** — a large win on big projects, where parsing (not +> the SQL) dominates each task's time. No `git add -f` needed: because the runtime dbt is pinned to +> your local version (see "dbt version and the serverless environment" below), the version-specific +> msgpack always loads instead of being silently ignored. Deploy with `make deploy` (or run +> `make manifest` first) so the shipped msgpack always matches your current models. + +## Deploy and run + +``` +$ make deploy # regenerates the manifest + parse cache, then deploys +$ make run # == databricks bundle run dbt_factory_job +``` + +`make deploy` regenerates `target/manifest.json` and `target/partial_parse.msgpack` (via `dbt +parse`) before deploying, so the task graph and the synced parse cache always match your current +models. You can also call the CLI directly — just run `make manifest` first: + +``` +$ databricks bundle deploy --target dev +$ databricks bundle run dbt_factory_job +``` + +Open the run URL the CLI prints to watch the generated per-model task graph execute. Deploying +in `dev` mode prefixes resources with `[dev your_name]` and pauses the daily schedule; deploy to +`prod` with `--target prod`. + +## Configuring the generated job + +A few knobs are exposed as constants at the top of `resources/__init__.py`: + +* `BUNDLE_TESTS` — when `True`, single-model tests are bundled into one `dbt test` task per + resource (fewer task startups; faster for test-heavy projects). Default `False` (one task per + test node, for maximum per-test visibility). +* `ENVIRONMENT_KEY` — the serverless environment key (default `Default`). +* `EXTRA_DBT_COMMAND_OPTIONS` — extra options appended to every generated dbt command. + +The dbt target, warehouse, catalog, and schema are configured in `dbt_profiles/profiles.yml` +and selected per bundle target via `--target ${bundle.target}`. + +### dbt version and the serverless environment + +You don't set the runtime dbt version by hand. At deploy time `resources/__init__.py` pins the +serverless environment to the **exact `dbt-databricks` version installed in the bundle's `.venv`** — +the same version you use locally to generate the manifest and develop with. `pyproject.toml` is the +single source of truth: change the version there, re-run `make setup`, and the next deploy uses it. +This guarantees the version running in Databricks matches the one you tested with. + +The version is shipped as a small `dbt_serverless_env.yaml` [base environment](https://docs.databricks.com/aws/en/compute/serverless/dependencies) +that the bundle generates and syncs on every deploy (git-ignored), so Databricks pre-builds the +environment once instead of installing dbt on every task. + +## Migrating an existing dbt project + +Bring your own dbt project by **generating a fresh project from the template and moving your dbt +files into it.** You don't touch dependencies, the vendored factory, or any paths — the generated +project already ships all of that. + +1. Generate a new project (or copy this `dbt_factory` example): + + ``` + $ databricks bundle init https://github.com/databricks/bundle-examples --template-dir contrib/templates/dbt-factory + ``` + +2. Remove the starter models and copy your dbt sources into the matching `src/` subdirectories: + + ``` + $ rm -r src/models/example + # Copy whichever of these your project has (skip the ones you don't use): + $ cp -R /path/to/your/dbt/models/* src/models/ + $ cp -R /path/to/your/dbt/seeds/* src/seeds/ + $ cp -R /path/to/your/dbt/snapshots/* src/snapshots/ + $ cp -R /path/to/your/dbt/macros/* src/macros/ + $ cp -R /path/to/your/dbt/tests/* src/tests/ + ``` + + The generated `dbt_project.yml` already points `model-paths`, `seed-paths`, etc. at these + `src/` folders, so your files are picked up as-is. Merge any model/seed configuration from your + own `dbt_project.yml` into the generated one (keep the generated `name`/`profile`), and remove + the leftover `models: dbt_factory: example:` block that referenced the deleted starter models — + otherwise `dbt parse` warns that those config paths don't apply to any resource. If you use dbt + packages, copy your `packages.yml` to the project root too: `make manifest` installs them + (`dbt deps`) and the bundle syncs the resulting `dbt_packages/` to the workspace, so the job + never installs packages at runtime. + +3. Point `dbt_profiles/profiles.yml` at your warehouse (`http_path`, `catalog`, `schema`). Leave + the `host`/`token` lines as they are — the runner notebook sets those at runtime. + +4. Generate the manifest and deploy: + + ``` + $ make setup + $ make manifest # dbt parse -> target/manifest.json + $ databricks bundle deploy --target dev + ``` + +That's the whole migration: no dependency wrangling and no path edits, because your project keeps +the generated layout (dbt project at the bundle root, factory under `src/`). If you'd rather keep +your project's existing directory structure instead of `src/`, edit the `*-paths` in +`dbt_project.yml` to point at your folders — nothing else changes. + +## Tests + +``` +$ make test # == uv run pytest tests +``` + +This runs the factory's unit tests plus an offline test that exercises the PyDABs integration +against the committed manifest; no workspace is required. One test compares the generated tasks +with a saved snapshot (`tests/test_data/expected_tasks.json`), so unintended changes to the +generated job fail the suite. After an intentional change to the generated output, refresh the +snapshot with `make test-update-expected-tasks`. + +There is also a live end-to-end test that generates a project from the template, deploys it to +your workspace, runs the generated job, verifies the output tables, and tears everything down +again: + +``` +$ make test-e2e +``` + +See [`tests/e2e/README.md`](tests/e2e/README.md) for the required environment variables. + +## Local development with dbt + +You can still develop the dbt project locally with the dbt CLI. Initialize your own profile with +`dbt init` (see `profile_template.yml`), then use `dbt run`, `dbt test`, etc. as usual. See the +[`dbt_sql`](../../dbt_sql) example for a more detailed local-dbt walkthrough. diff --git a/contrib/dbt_factory/databricks.yml b/contrib/dbt_factory/databricks.yml new file mode 100644 index 00000000..93b2db7e --- /dev/null +++ b/contrib/dbt_factory/databricks.yml @@ -0,0 +1,53 @@ +# This is a Declarative Automation Bundles definition for dbt_factory. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +# +# The job for this bundle is NOT defined in YAML. Instead it is generated at deploy time +# from the dbt manifest by resources/__init__.py (see the `python.resources` hook below), +# producing one Databricks task per dbt object (model / seed / snapshot / test). +bundle: + name: dbt_factory + uuid: 19ecc815-cff0-449c-91c1-e68239d49ccb + +# PyDABs: the Databricks CLI calls `load_resources` during `bundle deploy` to build resources +# defined in Python. See resources/__init__.py. +python: + venv_path: .venv + resources: + - "resources:load_resources" + +# These files are generated (by resources/__init__.py / `make manifest`) and git-ignored; include +# them explicitly so `bundle deploy` still syncs them to the workspace: +# - dbt_serverless_env.yaml the base_environment the generated job points at. +# - target/partial_parse.msgpack dbt's parse cache; each task injects it to skip parsing (see +# README). Regenerated by `make manifest`; usable at runtime +# because the runtime dbt is pinned to your local version. +# - dbt_packages/** dbt packages installed by `make manifest` (dbt deps), so tasks +# never install packages at runtime. +sync: + include: + - dbt_serverless_env.yaml + - target/partial_parse.msgpack + - dbt_packages/** + +# Deployment targets. The dbt target is selected via `--target ${bundle.target}`, so these +# names match the dbt outputs in dbt_profiles/profiles.yml. +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: https://company.databricks.com + + prod: + mode: production + workspace: + host: https://company.databricks.com + # We explicitly deploy to /Workspace/Users/user@company.com to make sure we only have a single copy. + root_path: /Workspace/Users/user@company.com/.bundle/${bundle.name}/${bundle.target} + permissions: + - user_name: user@company.com + level: CAN_MANAGE diff --git a/contrib/dbt_factory/dbt_packages/.gitkeep b/contrib/dbt_factory/dbt_packages/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/dbt_profiles/profiles.yml b/contrib/dbt_factory/dbt_profiles/profiles.yml new file mode 100644 index 00000000..45b37392 --- /dev/null +++ b/contrib/dbt_factory/dbt_profiles/profiles.yml @@ -0,0 +1,36 @@ +# This file defines dbt profiles for deployed dbt jobs. +# The generated Databricks job selects the target via `--target ${bundle.target}`, +# so the dbt target names below (dev / prod) match the bundle targets in databricks.yml. +dbt_factory: + target: dev # default target + outputs: + + # Doing local development with the dbt CLI? + # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init' + # (See README.md) + + # The default target when deployed with the Databricks CLI + dev: + type: databricks + method: http + catalog: catalog + schema: default + + http_path: /sql/1.0/warehouses/abcdef1234567890 + + # The workspace host / token are provided by the runner notebook at runtime + # (see src/databricks_dbt_factory/notebook/run_dbt_command.py). + host: "{{ env_var('DBT_HOST', '') }}" + token: "{{ env_var('DBT_ACCESS_TOKEN', '') }}" + + # The production target when deployed with the Databricks CLI + prod: + type: databricks + method: http + catalog: catalog + schema: default + + http_path: /sql/1.0/warehouses/abcdef1234567890 + + host: "{{ env_var('DBT_HOST', '') }}" + token: "{{ env_var('DBT_ACCESS_TOKEN', '') }}" diff --git a/contrib/dbt_factory/dbt_project.yml b/contrib/dbt_factory/dbt_project.yml new file mode 100644 index 00000000..63de848b --- /dev/null +++ b/contrib/dbt_factory/dbt_project.yml @@ -0,0 +1,28 @@ +name: 'dbt_factory' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'dbt_factory' + +# These configurations specify where dbt should look for different types of files. +# Everything dbt-related lives under src/ so the project can also hold non-dbt resources +# (such as the vendored databricks_dbt_factory library under src/databricks_dbt_factory). +model-paths: ["src/models"] +analysis-paths: ["src/analyses"] +test-paths: ["src/tests"] +seed-paths: ["src/seeds"] +macro-paths: ["src/macros"] +snapshot-paths: ["src/snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +# In this example config, we tell dbt to build all models in the example/ +# directory as views by default. These settings can be overridden in the +# individual model files using the `{{ config(...) }}` macro. +models: + dbt_factory: + example: + +materialized: view diff --git a/contrib/dbt_factory/profile_template.yml b/contrib/dbt_factory/profile_template.yml new file mode 100644 index 00000000..f26ec0b3 --- /dev/null +++ b/contrib/dbt_factory/profile_template.yml @@ -0,0 +1,23 @@ +# This file defines prompts with defaults for dbt initialization. +# It is used when the `dbt init` command is invoked. +# +fixed: + type: databricks +prompts: + host: + default: company.databricks.com + token: + hint: 'personal access token to use, dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + hide_input: true + http_path: + hint: 'HTTP path of SQL warehouse to use' + default: /sql/1.0/warehouses/abcdef1234567890 + catalog: + hint: 'initial catalog' + default: catalog + schema: + hint: 'personal schema where dbt will build objects during development, example: user_name' + threads: + hint: 'threads to use during development, 1 or more' + type: 'int' + default: 4 diff --git a/contrib/dbt_factory/pyproject.toml b/contrib/dbt_factory/pyproject.toml new file mode 100644 index 00000000..f2f53c93 --- /dev/null +++ b/contrib/dbt_factory/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "dbt_factory" +version = "0.0.1" +requires-python = ">=3.10,<3.13" +# Used at `bundle deploy` time by resources/__init__.py (to write the serverless +# base-environment file) and by the vendored databricks_dbt_factory core. +dependencies = [ + "PyYAML~=6.0.1", +] + +[dependency-groups] +dev = [ + # Required by the PyDABs hook (resources/__init__.py) at `bundle deploy` time. + "databricks-bundles==1.8.0", + # Single source of truth for the dbt-databricks version: used locally to generate the + # manifest (`make manifest`), and resources/__init__.py pins the serverless runtime to this + # exact installed version at deploy time. Pin/lock it for fully reproducible deploys. + "dbt-databricks>=1.8.0,<2.0.0", + "pytest", + "ruff", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +# Only the vendored library is packaged; the dbt project (src/models, ...) is not a package. +[tool.hatch.build.targets.wheel] +packages = ["src/databricks_dbt_factory"] + +[tool.ruff] +line-length = 120 diff --git a/contrib/dbt_factory/resources/__init__.py b/contrib/dbt_factory/resources/__init__.py new file mode 100644 index 00000000..4c6a7e5c --- /dev/null +++ b/contrib/dbt_factory/resources/__init__.py @@ -0,0 +1,194 @@ +""" +PyDABs integration for databricks-dbt-factory. + +At ``databricks bundle deploy`` time the Databricks CLI calls :func:`load_resources`, which +reads the dbt manifest and expands it into a Databricks job with one task per dbt object +(model / seed / snapshot / test) using the vendored ``databricks_dbt_factory`` core. No +per-model YAML is generated or checked in — the job graph is built on the fly from the manifest. + +This module is the only integration glue: it configures the vendored ``databricks_dbt_factory`` +core (under ``src/databricks_dbt_factory``) for serverless notebook tasks and returns a +``Resources`` object for PyDABs. +""" + +import os +from importlib.metadata import PackageNotFoundError, version + +import yaml +from databricks.bundles.core import Bundle, Resources +from databricks.bundles.jobs import Job + +from databricks_dbt_factory.dbt_factory import DbtFactory +from databricks_dbt_factory.dbt_task import DbtTaskOptions, TaskType +from databricks_dbt_factory.task_factory import ( + DbtDependencyResolver, + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, +) +from databricks_dbt_factory.utils import read_dbt_manifest + +# --- Configuration ------------------------------------------------------------------------- + +# Path to the dbt manifest, read at deploy time. Regenerate with `make manifest` (dbt parse). +# Override the location without editing this file via the DBT_MANIFEST_PATH env var. +MANIFEST_PATH = os.environ.get("DBT_MANIFEST_PATH", "target/manifest.json") + +# Name of the generated Databricks job. +JOB_NAME = "dbt_factory_job" + +# Key of the serverless environment (defined on the job below). +ENVIRONMENT_KEY = "Default" + +# Bundle single-model tests per resource into one `dbt test` task (fewer task startups, +# faster end-to-end runtime for projects with many tests). See the databricks-dbt-factory +# README "DBT Tests handling" section for the trade-offs. +BUNDLE_TESTS = False + +# Extra options appended to every generated dbt command (e.g. "--full-refresh"). Empty by default. +# Selection options (`--select`, `--exclude`, ...) and parse-context options (`--vars`, `--target`, +# `--profiles-dir`, `--project-dir`) are rejected: the factory owns selection and the runtime parse +# context must match the supplied manifest. +EXTRA_DBT_COMMAND_OPTIONS = "" + +# Serverless base-environment file, generated at deploy time from the pinned +# dbt-databricks version and synced with the bundle. The generated job points its +# serverless environment at this file (see build_job) so Databricks pre-builds the +# environment once instead of installing dbt on every task. `${workspace.file_path}` +# is substituted by the CLI at deploy time to the bundle's synced files root. +SERVERLESS_ENV_FILE = "dbt_serverless_env.yaml" +SERVERLESS_ENV_PATH = "${workspace.file_path}/" + SERVERLESS_ENV_FILE + +# The runner notebook shipped with the core, referenced in place (path relative to the bundle +# root). `PROJECT_DIRECTORY` is the path from the notebook's own directory back up to the dbt +# project root (== bundle root), where the runner changes directory to before running dbt. It is +# derived from the notebook path so the two stay consistent if the layout ever changes. +RUNNER_NOTEBOOK_PATH = "src/databricks_dbt_factory/notebook/run_dbt_command.py" +PROJECT_DIRECTORY = os.path.relpath(".", os.path.dirname(RUNNER_NOTEBOOK_PATH)) +PROFILES_DIRECTORY = "dbt_profiles" + + +def _build_tasks(target: str) -> list[dict]: + """Reads the dbt manifest and returns the list of Databricks task dicts (one per dbt node).""" + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + environment_key=ENVIRONMENT_KEY, + notebook_path=RUNNER_NOTEBOOK_PATH, + project_directory=PROJECT_DIRECTORY, + profiles_directory=PROFILES_DIRECTORY, + task_type=TaskType.NOTEBOOK, + ) + dbt_options = f"--target {target} {EXTRA_DBT_COMMAND_OPTIONS}".strip() + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + factory = DbtFactory(task_factories, bundle_tests=BUNDLE_TESTS) + manifest = read_dbt_manifest(MANIFEST_PATH) + return factory.create_tasks(manifest) + + +def _dbt_databricks_dependency() -> str: + """Pin the runtime to the exact dbt-databricks installed in this bundle's venv — the same + version used to generate the manifest (`make manifest`) and to develop locally. pyproject.toml + is therefore the single source of truth for the dbt version, and the version that runs in + Databricks is guaranteed to match the one you tested with (no separate range that can drift). + Local or dev builds (e.g. `1.9.0+custom`) are rejected: pip cannot resolve them from PyPI + when Databricks builds the serverless environment, which would fail every task at runtime. + """ + try: + installed = version("dbt-databricks") + except PackageNotFoundError as exc: + raise RuntimeError( + "dbt-databricks is not installed in the bundle venv — run `make setup` before deploying." + ) from exc + + from packaging.version import InvalidVersion, Version + + try: + parsed = Version(installed) + except InvalidVersion: + parsed = None + if parsed is None or parsed.local or parsed.is_devrelease: + raise RuntimeError( + f"The installed dbt-databricks version ({installed}) is not a plain PyPI release, so the " + "serverless environment cannot install it. Set a released version in pyproject.toml and " + "re-run `make setup` before deploying." + ) + return f"dbt-databricks=={installed}" + + +def _serverless_environment_spec() -> dict: + """Contents of the base-environment file: the environment version plus the pinned + dbt-databricks dependency. + + Only dbt-databricks is listed. The runner notebook (notebook/run_dbt_command.py) also imports + databricks-sdk, but that is a transitive dependency of dbt-databricks, so pinning + dbt-databricks pulls it into the serverless environment automatically — no separate entry + needed. If you ever build a task environment without dbt-databricks, add databricks-sdk here + too, or the runner fails at import.""" + return {"environment_version": "4", "dependencies": [_dbt_databricks_dependency()]} + + +def _write_serverless_environment_file() -> None: + """Write the base-environment file to the bundle root so `bundle deploy` syncs it to the + workspace, where the generated job references it via base_environment. The write is skipped + when the file already has the desired content, so commands like `bundle validate` stay free + of side effects once the file exists.""" + content = yaml.safe_dump(_serverless_environment_spec(), sort_keys=False) + try: + with open(SERVERLESS_ENV_FILE, "r", encoding="utf-8") as f: + if f.read() == content: + return + except OSError: + pass + try: + with open(SERVERLESS_ENV_FILE, "w", encoding="utf-8") as f: + f.write(content) + except OSError as exc: + raise RuntimeError( + f"Cannot write {SERVERLESS_ENV_FILE}, which the generated job needs as its serverless " + f"base environment: {exc}" + ) from exc + + +def build_job(target: str) -> Job: + """Builds the Databricks job that runs the dbt project on serverless via notebook tasks.""" + return Job.from_dict( + { + "name": JOB_NAME, + "queue": {"enabled": True}, + "trigger": { + # Run this job every day, exactly one day from the last run; + # see https://docs.databricks.com/api/workspace/jobs/create#trigger + "periodic": {"interval": 1, "unit": "DAYS"}, + }, + "tasks": _build_tasks(target), + "environments": [ + { + "environment_key": ENVIRONMENT_KEY, + # Pre-built base env (synced with the bundle), not inline deps — + # the two are mutually exclusive; the file pins dbt-databricks. + "spec": {"base_environment": SERVERLESS_ENV_PATH}, + } + ], + } + ) + + +def load_resources(bundle: Bundle) -> Resources: + """ + Called by the Databricks CLI during `bundle deploy` to load resources defined in Python. + + Reads the dbt manifest, writes the serverless base-environment file (synced with the bundle), + and registers the generated job. After deployment this function is not used. + """ + _write_serverless_environment_file() + resources = Resources() + resources.add_job(JOB_NAME, build_job(bundle.target)) + return resources diff --git a/contrib/dbt_factory/src/analyses/.gitkeep b/contrib/dbt_factory/src/analyses/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/__init__.py b/contrib/dbt_factory/src/databricks_dbt_factory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/__version__.py b/contrib/dbt_factory/src/databricks_dbt_factory/__version__.py new file mode 100644 index 00000000..a8d4557d --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/__version__.py @@ -0,0 +1 @@ +__version__ = "0.3.5" diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/dbt_factory.py b/contrib/dbt_factory/src/databricks_dbt_factory/dbt_factory.py new file mode 100644 index 00000000..39cc0788 --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/dbt_factory.py @@ -0,0 +1,1347 @@ +import heapq +from collections.abc import Iterator +from dataclasses import dataclass, field, replace +from pathlib import PurePosixPath, PureWindowsPath +from typing import cast + +from databricks_dbt_factory.dbt_task import DbtTask +from databricks_dbt_factory.task_factory import TaskFactory, TestTaskFactory +from databricks_dbt_factory.utils import DYNAMIC_VALUE_REFERENCE, build_task_key_maps + +# The `unique_id` prefixes of resources a test can be attached to. +_DBT_TEST_TARGET_PREFIXES = ("model.", "seed.", "snapshot.", "source.") + + +def _flatten_fqn(fqn: list[str]) -> list[str]: + """ + Flattens an fqn the way dbt does before comparing it, splitting every segment on `.`. + + `is_selected_node` treats dots inside a segment as namespace separators, so a test named + `check.nested` with fqn `['probe', 'check.nested']` compares as `['probe', 'check', 'nested']` — + which is why the shorter `probe.check` matches it as a subtree parent. + """ + return [part for segment in fqn for part in segment.split(".")] + + +@dataclass +class _SelectionPlan: + """A dbt selector paired with the indirect-selection mode needed to keep it exact.""" + + select: str + indirect_selection: str + + +@dataclass +class _Gating: + """ + What deciding a node's gating test edges needs, in per-test mode. + + `tests` indexes each test under its referenced resources, `ancestors` records strict dbt ancestors, + `resources_by_task_key` maps immediate emitted dependencies back to manifest resource ids, and + `eligible_test_keys` memoizes the tests eligible at each resource's downstream frontier. + """ + + tests: dict[str, list[tuple[str, frozenset[str]]]] = field(default_factory=dict) + ancestors: dict[str, set[str]] = field(default_factory=dict) + resources_by_task_key: dict[str, str] = field(default_factory=dict) + eligible_test_keys: dict[str, frozenset[str]] = field(default_factory=dict) + + +def _candidate_file_names(original_file_path: str) -> frozenset[str]: + """Returns every base name implied by POSIX and Windows path semantics.""" + return frozenset( + { + PurePosixPath(original_file_path).name, + PureWindowsPath(original_file_path).name, + } + ) + + +def _base_file_name(original_file_path: str) -> str: + """ + Returns the path-flavour-invariant base name dbt's `file:` selector matches. + + dbt assembles `original_file_path` with `os.path.join`/`os.path.relpath` and `FileSelectorMethod` + splits it with `Path(...).name`, so a path containing a backslash can mean either a Windows hierarchy + or a literal POSIX file name. Manifest JSON carries no producer-platform metadata. A `file:` term + is therefore emitted only when both path flavours yield the same base name; otherwise the remaining + selector terms must prove exactness or generation refuses the resource. + """ + candidates = _candidate_file_names(original_file_path) + return next(iter(candidates)) if len(candidates) == 1 else "" + + +class DbtFactory: + """A factory for generating Databricks job definitions from dbt manifests.""" + + def __init__( + self, + task_factories: dict[str, TaskFactory], + bundle_tests: bool = False, + ): + """ + Initializes the dbt factory. + + Args: + task_factories (dict[str, TaskFactory]): Maps dbt resource types (`model`, `seed`, + `snapshot`, `test`) to their respective `TaskFactory` instances. Omitting `test` + disables test-task generation entirely. + bundle_tests (bool): When True, emit one bundled `_test` task per tested + resource and rewire downstream models/seeds/snapshots to depend on the upstream's + bundled test task so failing tests halt the DAG. When False, emit one task per + dbt test node. + """ + self.task_factories = task_factories + self.bundle_tests = bundle_tests + + def create_tasks(self, dbt_manifest: dict) -> list[dict]: + """ + Generates the Databricks task dictionaries from a dbt manifest. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[dict]: Task dictionaries ready to be injected into the `tasks` list of a + Databricks job spec. + + Raises: + ValueError: If the generated job would exceed Databricks' 1,000-task limit. + """ + tasks = self._create_tasks(dbt_manifest) + if len(tasks) > 1_000: + raise ValueError( + f"Databricks jobs support at most 1,000 tasks; this manifest generates {len(tasks):,}. " + f"Reduce the generated resources or enable test bundling." + ) + return [task.to_dict() for task in tasks] + + _GATEABLE_TYPES = frozenset({"model", "seed", "snapshot"}) + _DBT_TEST_TARGET_PREFIXES = _DBT_TEST_TARGET_PREFIXES + + # Characters that still change how an *explicit* `fqn:` selector is interpreted, so a component + # containing one cannot be used to address a node. Each verified against dbt 1.12.0 with `dbt ls`. + # + # ' ' union separator (`graph/cli.py` splits the raw spec on spaces) — the fragments then match + # independently. dbt rejects a space in a resource *name* outright, but a space in a + # *directory* reaches the fqn, and `fqn:probe.my dir.orders` selects nothing. + # ',' intersection separator — the component is read as two, matching zero nodes + # '*?[]' `fnmatch` pattern syntax, still honoured after the `fqn:` prefix: `fqn:probe.a*star` + # selects `a*star` *and* `aXstar`, so the glob is live rather than literal + # + # Databricks substitutes complete `{{...}}` dynamic value references in task commands before the + # task runs. Those references are screened separately; literal single braces and incomplete pairs + # remain ordinary selector text under `fqn:`. + # + # Emitting `fqn:` rather than a bare value is what keeps this list short. dbt's + # `SelectionCriteria.default_method` dispatches a bare value containing `/` to `MethodName.Path` and + # one ending `.sql`/`.py`/`.csv` to `MethodName.File`, both of which then match nothing; naming the + # method explicitly bypasses that heuristic entirely. Verified: bare `probe.orders.sql` and + # `probe.check/slash` select nothing, while `fqn:probe.orders.sql` and `fqn:probe.check/slash` each + # resolve to exactly their node. `:` and literal braces are likewise literal under `fqn:`. + _SELECTOR_METACHARACTERS = frozenset(" ,*?[]") + + @classmethod + def _node_select( + cls, + node_info: dict, + source_info: dict | None = None, + peers: dict | None = None, + ) -> str: + """ + Returns the dbt `--select` argument addressing exactly one node. + + `peers` is every selectable resource in the manifest, keyed by id, used to prove the finished + selector matches only `node_info` (see `_assert_exact`). It is optional so the library API and + unit tests can build a selector for a lone node, but callers with a manifest should pass it: + without it, exactness is assumed rather than established. + + Every node is addressed the same way — the intersection (`,`, dbt's AND) of every independent + fact the manifest gives us about it: + + ,package:,file:[,test_name:] + + No term is exact alone, which is why they are combined rather than chosen between: + + * the **fqn** is matched as a positional path *prefix*, so it also selects nodes nested + beneath it — and dbt compares a node's fqn with its package stripped too, so a package's + `models/probe/alpha.sql` is matched by the root project's `pkg.alpha` as well; + * **`package:`** narrows to the owning package, but matches all of it; + * **`file:`** narrows to one source file, but matches that base name in every package, and a + `schema.yml` is shared by every test declared in it; + * **`test_name:`** narrows a generic test to its type (`not_null`, `unique`), separating tests + that share a `schema.yml` — but only when their types differ, so two `not_null` tests in one + file still need the bare **name**, which dbt matches against the fqn's leaf. The name stands + in for the fqn when the fqn itself is unusable. + + A term is omitted when dbt cannot express it literally or when combining it with the other + terms would form a Databricks dynamic value reference. Exactness is rechecked against `peers` + after any omission. `source_info` addresses a source's tests, which dbt selects by + `source:..` rather than by fqn. + + **The fqn or the bare name must survive, and the result must then be checked for exactness.** + + dbt has no `unique_id:` selector method, so a selector is always a *predicate*, and its + exactness has to be established rather than assumed. Requiring the fqn or the bare name is + necessary — `package:`, `file:`, `resource_type:` and `test_name:` each address a group — but + is not sufficient. An fqn is a positional prefix over dbt's *flattened* fqn, so it names a + subtree, not a node: + + * a test named `check.nested` flattens to `[probe, check, nested]`, so the sibling `check`'s + selector `probe.check` matches it too — even with `package:`, `file:` and `test_name:` all + present and identical; + * a unit test on `orders` (`[probe, orders, unit_orders]`) and a data test named + `orders.unit_orders` (`[probe, orders.unit_orders]`) flatten identically, which `resource_type:` + does separate; + * dbt clones a unit test per model version leaving both clones the same fqn, name and file; + * two generic tests may simply *share* an fqn: dbt does not require test names to be unique and + disambiguates in the `unique_id` hash only. + + `_assert_exact` checks direct selectors against the manifest. Test selectors whose direct form is + ambiguous may instead use `_test_selection_plan`, which scopes the selector to one exact parent. + + Exactness is checked per node against the rest of the manifest because it is a property of the + node and its selectable neighbours, not of one manifest entry in isolation. + + Raises: + ValueError: when neither the fqn nor the bare name survives, when the final selector cannot + avoid a Databricks dynamic value reference, or when it also matches another resource. + """ + if source_info is not None: + return cls._source_select(source_info) + + # Exactly one of the fqn and the bare name is used, and one of them must be: they are the only + # terms that address a *single* resource, so a selector without either could run another task's + # resource. dbt offers no `unique_id:` method — the full set it accepts, confirmed against dbt + # 1.12.0, is `fqn`, `tag`, `source`, `path`, `file`, `package`, `config`, `test_name`, + # `test_type`, `resource_type`, `state`, `exposure`, `metric`, `result`, `source_status`, + # `group`, `version`, `access`, `semantic_model`, `saved_query`, `unit_test`, `selector`. + terms: list[str] = [] + fqn = node_info.get("fqn") or [] + # The `fqn:` prefix does *not* neutralise graph operators: `fqn:probe.orders+1` still selects + # `orders` and its children, verified with `dbt ls` on dbt 1.12.0. The boundary check therefore + # applies to the joined value; naming the method only bypasses the dispatch heuristic. + if fqn and cls._is_usable_selector(".".join(fqn)) and all(cls._is_usable_component(part) for part in fqn): + # `fqn:` names the method explicitly so `/` and `.sql` remain literal fqn text instead of + # dispatching to path or file matching. `fqn:probe.orders.sql` and + # `fqn:probe.check/slash` each resolve to exactly their node on dbt 1.12.0. + terms.append(f"fqn:{'.'.join(fqn)}") + elif cls._is_usable_component(name := node_info.get("name") or "") and cls._is_usable_selector(name): + # The fqn is unusable, so fall back to the bare resource name, which dbt matches against + # the fqn's leaf. It is the only term that tells apart two nodes sharing a package, a + # file and a test type — two `not_null` tests in one `schema.yml`, say. Not used *with* a + # usable fqn, whose leaf already carries it. + terms.append(f"fqn:{name}") + else: + raise cls._unaddressable(node_info) + + package = node_info.get("package_name") or "" + if cls._is_usable_component(package): + terms.append(f"package:{package}") + + file_name = cls._base_file_name(node_info.get("original_file_path") or "") + if cls._is_usable_component(file_name): + terms.append(f"file:{file_name}") + + # `resource_type:` separates nodes whose flattened fqns coincide across *types*, which the + # other terms cannot: a unit test `unit_orders` on `orders` (fqn [probe, orders, unit_orders]) + # and a data test named `orders.unit_orders` (fqn [probe, orders.unit_orders]) flatten to the + # same string and share their file. Confirmed on dbt 1.12.0: adding `resource_type:unit_test` + # and `resource_type:test` respectively isolates each to one node. + # + # It does *not* undo dbt's indirect selection: `resource_type:test` still admits the attached + # tests eagerly pulled in with a selected model, because dbt intersects sets that are already + # expanded. Suppressing that expansion is the selection plan's job, not this term's: direct test + # plans pin `--indirect-selection empty`, and parent-scoped cautious plans are accepted only when + # `_eager_expansion_superset` proves the expanded result is the intended test alone. + resource_type = node_info.get("resource_type") or "" + if cls._is_usable_component(resource_type): + terms.append(f"resource_type:{resource_type}") + + test_name = (node_info.get("test_metadata") or {}).get("name") or "" + if cls._is_usable_component(test_name): + terms.append(f"test_name:{test_name}") + + select = cls._compose_selector_terms(terms, node_info) + if peers is not None: + cls._assert_exact(select, node_info, peers) + return select + + @classmethod + def _compose_selector_terms(cls, terms: list[str], node_info: dict) -> str: + """Joins terms while preserving the required first term and omitting unsafe optional terms.""" + safe_terms = list(terms) + while match := DYNAMIC_VALUE_REFERENCE.search(select := ",".join(safe_terms)): + offset = 0 + closing_term = 0 + for index, term in enumerate(safe_terms): + offset += len(term) + if match.end() <= offset: + closing_term = index + break + offset += 1 + if closing_term == 0: + raise cls._dynamic_reference(node_info, select) + safe_terms.pop(closing_term) + return select + + @classmethod + def _assert_no_dynamic_reference(cls, select: str, node_info: dict) -> None: + """Rejects a complete dynamic value reference in a final Databricks task selector.""" + if DYNAMIC_VALUE_REFERENCE.search(select): + raise cls._dynamic_reference(node_info, select) + + @classmethod + def _test_selection_plan(cls, test_info: dict, peers: dict) -> _SelectionPlan: + """ + Builds an exact plan for one direct test task. + + An already-exact test selector runs with indirect selection disabled. If the direct selector + matches multiple tests, a test with exactly one manifest parent can be isolated by intersecting + the parent's exact selector with the test selector under `cautious`, provided the conservative + per-term expansion proof shows that the finished selector can run only the intended test. + """ + select = cls._node_select(test_info) + intended = cls._own_ids(test_info, peers) + selected = set(cls._matching_ids(select, peers)) + if selected == intended: + return _SelectionPlan(select, "empty") + + surplus = selected - intended + missing = intended - selected + if missing or not surplus: + cls._assert_exact(select, test_info, peers) + + parents = cls._test_parent_ids(test_info) + if len(parents) != 1: + cls._assert_exact(select, test_info, peers) + parent_id = next(iter(parents)) + if parent_id not in peers: + cls._assert_exact(select, test_info, peers) + parent_info = peers[parent_id] + parent_select = cls._node_select( + parent_info, + source_info=parent_info if parent_info.get("resource_type") == "source" else None, + peers=peers, + ) + scoped_select = f"{parent_select},{select}" + cls._assert_no_dynamic_reference(scoped_select, test_info) + if cls._eager_expansion_superset(scoped_select, peers) != intended: + cls._assert_exact(select, test_info, peers) + return _SelectionPlan(scoped_select, "cautious") + + @classmethod + def _eager_expansion_superset(cls, select: str, peers: dict) -> set[str]: + """ + Returns a conservative superset of a cautious selector's result. + + dbt expands every comma-separated term before intersecting the term results. For each term this + proof uses eager expansion: its direct matches plus every test with any directly matched parent. + Cautious expansion can only admit fewer tests, so intersecting these eager term sets remains a + superset of the finished cautious result. A parent-scoped plan is accepted only when that superset + is exactly the intended test. + + The intended single-parent test is present by construction: every test term matches it directly, + and every exact-parent term matches its sole parent and therefore adds the test indirectly. + """ + possible: set[str] | None = None + for term in select.split(","): + direct = set(cls._matching_ids(term, peers)) + expanded = direct | cls._tests_attached_to_any(direct, peers) + possible = expanded if possible is None else possible & expanded + return possible or set() + + @classmethod + def _tests_attached_to_any(cls, parent_ids: set[str], peers: dict) -> set[str]: + """Returns enabled test ids having at least one manifest parent in `parent_ids`.""" + if isinstance(peers, _SelectorIndex): + return peers.tests_attached_to_any(parent_ids) + return { + full_name + for full_name, info in peers.items() + if info.get("resource_type") in {"test", "unit_test"} and cls._test_parent_ids(info) & parent_ids + } + + @classmethod + def _test_parent_ids(cls, test_info: dict) -> frozenset[str]: + """Returns every model, seed, snapshot, or source dependency id, including absent resources.""" + return frozenset( + dep + for dep in test_info.get("depends_on", {}).get("nodes", []) + if dep.startswith(cls._DBT_TEST_TARGET_PREFIXES) + ) + + @classmethod + def _assert_exact(cls, select: str, node_info: dict, peers: dict) -> None: + """ + Raises unless `select` runs only the node it was built for. + + Direct test plans pin `--indirect-selection empty`, so their exactness is a plain intersection of + the emitted terms. Parent-scoped cautious plans apply their additional proof in + `_test_selection_plan`. Bundled tasks reuse these per-test plans and union only plans with the + same indirect-selection mode. + + The contract is **equality**, not "no surplus". A selector that matches nothing is just as wrong as + one that matches too much: `dbt test` and `dbt run` both exit 0 on a zero-match selector, so the + task would go green having asserted or built nothing. + + The node recognises itself by object identity *or* by `unique_id`: identity alone would call a + node its own collision if a caller passed a copy, and `unique_id` alone fails on hand-written + fixtures that omit the field. + """ + allowed = cls._own_ids(node_info, peers) + run = set(cls._matching_ids(select, peers)) + surplus = run - allowed + if surplus: + raise cls._ambiguous(node_info, select, sorted(surplus)) + missing = allowed - run + if missing: + raise cls._selects_nothing(node_info, select, sorted(missing)) + + @staticmethod + def _own_ids(node_info: dict, peers: dict) -> set[str]: + """ + The `peers` keys that *are* this node, by `unique_id` when present or by object identity. + + `unique_id` is a direct dict lookup, so the common path costs nothing. The identity scan is the + fallback for hand-written fixtures that omit the field — walking every peer here once per node was + itself quadratic, and defeated the index added to avoid exactly that. + """ + own_id = node_info.get("unique_id") + if own_id is not None and peers.get(own_id) is not None: + return {own_id} + return {full_name for full_name, info in peers.items() if info is node_info} + + @staticmethod + def _base_file_name(original_file_path: str) -> str: + """The base name dbt's `file:` selector matches — see the module-level `_base_file_name`.""" + return _base_file_name(original_file_path) + + @classmethod + def _source_select(cls, source_info: dict) -> str: + """ + Returns the `source:..
` selector for a source's tests. + + A source has no fqn selector of its own, so all three parts must be usable: unlike a node + there is no other term to fall back on. + + `.` is disallowed on top of the usual metacharacters. It delimits this grammar, which accepts + at most three parts, so a dot inside one makes four and dbt rejects the whole selector with a + Runtime Error ("Invalid source selector value") rather than merely selecting nothing — + confirmed against dbt 1.12.0 with `dbt ls`. A node's fqn is unaffected: there the dot is the + separator we are already building with, and a dotted segment stays addressable. + + Complete Databricks dynamic value references are checked after assembly, including references + completed by the separator between the source and table. None of the three source parts can be + omitted, so such a selector is refused with the unsafe final value in the error. + + The finished string is checked with `_is_usable_selector` too: like a bare name, the whole + `source:...` is one raw selector, so a trailing `+N` on the table is read as a graph operator + and matches nothing while `dbt test` still exits 0. Only the boundary matters — + `source:pkg.raw.2+ord` resolves exactly — so the check goes on the assembled string rather + than on each part. + """ + package = source_info.get("package_name") or "" + source_name = source_info.get("source_name") or "" + table = source_info.get("name") or "" + select = f"source:{package}.{source_name}.{table}" + cls._assert_no_dynamic_reference(select, source_info) + parts = (package, source_name, table) + if not all(cls._is_usable_component(part) and "." not in part for part in parts): + raise cls._unaddressable(source_info) + if not cls._is_usable_selector(select): + raise cls._unaddressable(source_info) + return select + + @classmethod + def _is_usable_component(cls, value: str) -> bool: + """Whether `value` can appear inside a selector component and still mean itself.""" + return ( + bool(value) + and not cls._SELECTOR_METACHARACTERS & set(value) + and DYNAMIC_VALUE_REFERENCE.search(value) is None + ) + + @classmethod + def _is_usable_selector(cls, value: str) -> bool: + """ + Whether a fully assembled selector avoids dbt graph expansion and Databricks substitution. + + dbt reads a trailing `+N` as child depth even after an explicit method prefix, so + `fqn:probe.orders+1` selects `orders` and its child. Leading `@` and `N+` forms, and `+` inside + a segment, remain literal under an explicit `fqn:` method; `fqn:@weird`, `fqn:2+orders`, + `fqn:pkg.+leading`, and `fqn:pkg.raw.2+ord` each address their exact node on dbt 1.12.0. + + Complete `{{...}}` references are unusable because Databricks substitutes them before dbt receives + the command. The check applies after selector components are joined so references spanning + components are also rejected. + """ + return not value.rstrip("0123456789").endswith("+") and DYNAMIC_VALUE_REFERENCE.search(value) is None + + @staticmethod + def _flat_fqn(fqn: list[str]) -> list[str]: + """Flattens an fqn as dbt does — see the module-level `_flatten_fqn`.""" + return _flatten_fqn(fqn) + + @classmethod + def _fqn_term_matches(cls, term: str, node_info: dict) -> bool: + """ + Whether dbt's fqn selector `term` matches `node_info`. + + Mirrors `QualifiedNameSelectorMethod.node_is_match` on dbt 1.12.0, including the two details + that make an fqn a *predicate* rather than an identifier: + + * the comparison is a positional walk over the *flattened* fqn, and a selector that runs out of + parts still matches — so `probe.check` matches `['probe', 'check', 'nested']`, i.e. anything + nested beneath it, whether the nesting comes from a directory or from a dot in a name; + * it is retried with the node's own package stripped, so a package's `models/probe/alpha.sql` + is matched by the root project's `probe.alpha` too. + + Wildcards are not handled: `_is_usable_component` rejects `*?[]` outright, so no term reaching + here contains one. This mirrors dbt rather than reasoning about it because being *lenient* here + would mean accepting a selector that matches two nodes, which is the whole bug class; the + integration suite pins the mirror against `dbt ls`. + """ + fqn = node_info.get("fqn") or [] + if not fqn: + # A manifest missing `fqn` is not something dbt produces, but a hand-rolled or truncated one + # can be, and the selector then falls back to the bare name. dbt would still match that name + # against the fqn's leaf, so compare against the name rather than declining outright — + # otherwise the exactness check reports a selector that reaches nothing. + return bool(node_info.get("name")) and node_info.get("name") == term + # dbt's `is_versioned` requires the resource type to be a model (`VERSIONED_NODE_TYPES`), not + # merely that `version` is set — and a unit-test clone *does* carry `version`. Deriving it from + # the field alone takes dbt's versioned branch for unit tests and so skips its plain + # `fqn[-1] == term` match, making the mirror stricter than dbt and hiding a real collision on + # such a node's bare-name selector. + is_versioned = node_info.get("resource_type") == "model" and node_info.get("version") is not None + return cls._is_selected_node(fqn, term, is_versioned) or cls._is_selected_node(fqn[1:], term, is_versioned) + + @classmethod + def _is_selected_node(cls, fqn: list[str], term: str, is_versioned: bool) -> bool: + """Mirrors dbt's `is_selected_node` for the operator-free, wildcard-free terms we emit.""" + if not fqn: + return False + selector_parts = term.split(".") + if cls._matches_fqn_leaf(fqn, term, selector_parts, is_versioned): + return True + flat_fqn = cls._flat_fqn(fqn) + if len(flat_fqn) < len(selector_parts): + return False + for index, part in enumerate(selector_parts): + if flat_fqn[index] != part: + return False + return True + + @staticmethod + def _matches_fqn_leaf(fqn: list[str], term: str, selector_parts: list[str], is_versioned: bool) -> bool: + """ + Whether `term` matches the fqn's leaf, dbt's shortcut before the positional walk. + + A versioned model's fqn ends `[..., , v]`, and dbt lets the last two segments match on + either the `.` or `_` delimiter, so `orders`, `orders.v1` and `orders_v1` all hit + `['probe', 'orders', 'v1']`. + """ + if not is_versioned: + return fqn[-1] == term + if len(fqn) < 2: + return False + return fqn[-2] == term or "_".join(fqn[-2:]) == "_".join(selector_parts[-2:]) + + @classmethod + def _term_matches(cls, term: str, node_info: dict) -> bool: + """Whether one emitted selector term matches `node_info`.""" + method, _, value = term.partition(":") + if not _ or method == "fqn": + # A bare value is still accepted so a hand-written selector keeps working; everything the + # factory emits now carries the explicit `fqn:` method. + return cls._fqn_term_matches(value if method == "fqn" else term, node_info) + if method == "source": + parts = value.split(".") + return ( + len(parts) == 3 + and node_info.get("resource_type") == "source" + and (node_info.get("package_name") or "") == parts[0] + and (node_info.get("source_name") or "") == parts[1] + and (node_info.get("name") or "") == parts[2] + ) + if method == "package": + return (node_info.get("package_name") or "") == value + if method == "file": + # dbt's `FileSelectorMethod` matches the base name *or* its stem, so `file:a.yml` also + # matches a node declared in `a.yml.yml`. Mirroring only the name would let that collision + # past `_assert_exact` — confirmed with `dbt ls` on dbt 1.12.0, where the `a.yml` task's + # selector resolves to the `a.yml.yml` test as well. + for base in _candidate_file_names(node_info.get("original_file_path") or ""): + if value in (base, base.rsplit(".", 1)[0] if "." in base else base): + return True + return False + if method == "resource_type": + return (node_info.get("resource_type") or "") == value + if method == "test_name": + return ((node_info.get("test_metadata") or {}).get("name") or "") == value + return True # pragma: no cover - no other method is emitted + + @staticmethod + def _selector_index(peers: dict) -> dict: + """Wraps `peers` in the narrowing index used for exactness checks (see `_SelectorIndex`).""" + return _SelectorIndex(peers) + + @classmethod + def _matching_ids(cls, select: str, candidates: dict) -> list[str]: + """ + The ids in `candidates` that `select` matches, by intersecting every term as dbt does. + + This is a *direct* match: it does not model dbt's indirect selection, so it is only exactness + evidence for a plan that suppresses expansion. Direct test plans pin `--indirect-selection empty` + and so need nothing further; parent-scoped cautious plans are proved instead by + `_eager_expansion_superset`, which expands each term itself. + + `candidates` should be a `_SelectorIndex` on any real manifest: scanning every node for every + node is quadratic, and measurably so — 90 seconds for a 6,000-node manifest. The index narrows the + scan to the nodes sharing this selector's `package:` and `file:`, which is a handful. A plain dict + still works, for the unit tests and library callers that pass one. + """ + terms = select.split(",") + scan = candidates.narrow(terms) if isinstance(candidates, _SelectorIndex) else candidates + matched: list[str] = [] + for full_name, info in scan.items(): + if all(cls._term_matches(term, info) for term in terms): + matched.append(full_name) + return matched + + @staticmethod + def _ambiguous(node_info: dict, select: str, also_matched: list[str]) -> ValueError: + """ + Builds the error raised when a selector is valid but not exact. + + Distinct from `_unaddressable`: nothing about *this* resource's name is wrong, so the remedy is + about the collision with its neighbours rather than about selector syntax. + """ + name = node_info.get("name") + path = node_info.get("original_file_path") + return ValueError( + f"Cannot generate a task for {name!r} ({path}): the generated selector for it " + f"({select}) also runs {', '.join(sorted(also_matched))}. dbt has no unique-id selector, so " + f"the factory cannot prove a task addresses these separately. It does not search alternate " + f"spellings once a usable FQN exists, and refuses to risk running the others before their own " + f"dependencies have completed. Rename {name!r} so that its dotted " + f"name neither matches nor prefixes a " + f"sibling's, or move it to a file of its own." + ) + + @staticmethod + def _dynamic_reference(node_info: dict, select: str) -> ValueError: + """Builds the error for a selector Databricks would interpret as a dynamic value reference.""" + name = node_info.get("name") + path = node_info.get("original_file_path") + return ValueError( + f"Cannot generate a task for {name!r} ({path}): the final selector ({select}) contains a " + f"Databricks dynamic value reference. Rename the resource or file so its selector terms do " + f"not form a complete reference." + ) + + @staticmethod + def _selects_nothing(node_info: dict, select: str, missing: list[str]) -> ValueError: + """ + Builds the error raised when a selector fails to reach what the task is meant to run. + + Distinct from `_ambiguous`, which is the opposite failure. This one is easy to overlook because it + is silent at run time: `dbt test` and `dbt run` exit 0 on a selector that matches nothing, so the + task goes green having asserted or built nothing at all. + """ + name = node_info.get("name") + path = node_info.get("original_file_path") + return ValueError( + f"Cannot generate a task for {name!r} ({path}): the selector dbt offers for it ({select}) " + f"does not reach {', '.join(missing)}. dbt exits 0 for a selector that matches nothing, so " + f"the task would report success having run nothing. This is a bug in selector construction " + f"rather than something to fix in the project — please report it." + ) + + @staticmethod + def _unaddressable(node_info: dict) -> ValueError: + """ + Builds the error raised when no selector can address a node. + + This message is the whole of what a CLI user sees (`main` reports it without a traceback), so + it leads with the resource and the remedy and keeps the reasoning to one closing line. + """ + name = node_info.get("name") + path = node_info.get("original_file_path") + return ValueError( + f"Cannot generate a task for {name!r} ({path}): dbt cannot select it uniquely. " + f"Rename the resource or its file so that it does not end with a dbt graph operator (a " + f"trailing +N), contains none of a space, comma or one of *?[], and contains no complete " + f"Databricks dynamic value reference (`{{{{...}}}}`) — or, for a source, a dot. Without a " + f"usable name or path, the only terms left match a group of " + f"resources, so the task could run another task's resource." + ) + + def _create_tasks(self, dbt_manifest: dict) -> list[DbtTask]: + """ + Builds `DbtTask` instances from the manifest, applying the bundling and gating policies. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[DbtTask]: `DbtTask` instances (not yet rendered to dicts). + """ + dbt_nodes = self._enabled_only(dbt_manifest.get("nodes", {})) + dbt_sources = self._enabled_only(dbt_manifest.get("sources", {})) + dbt_unit_tests = self._enabled_only(dbt_manifest.get("unit_tests", {})) + + # All directly selectable resources participate in selector exactness and test selection-plan + # checks, including sources even though no emitted command builds a source directly. + peers = _SelectorIndex({**dbt_nodes, **dbt_unit_tests, **dbt_sources}) + + bundle = "test" in self.task_factories and self.bundle_tests + bundled_tests: dict[str, list[tuple[str, dict]]] = {} + standalone_tests: list[tuple[str, dict]] = [] + if bundle: + bundled_tests, standalone_tests = self._classify_tests(dbt_nodes, dbt_sources, dbt_unit_tests) + standalone_test_ids = {full_name for full_name, _ in standalone_tests} + + # Unit tests live under the manifest `unit_tests` key, not `nodes`. In per-test mode each + # gets its own task (those whose target model is absent are skipped), so include their ids + # here to receive a task key from `build_task_key_maps`. + unit_test_ids = ( + self._emitted_unit_test_ids(dbt_unit_tests, dbt_nodes) + if not bundle and "test" in self.task_factories + else [] + ) + + task_ids = [] + for full_name, info in dbt_nodes.items(): + if self._node_gets_own_task(full_name, info, bundle, standalone_test_ids): + task_ids.append(full_name) + task_ids += unit_test_ids + task_keys, bundled_test_keys = build_task_key_maps(task_ids, sorted(bundled_tests)) + + gating = _Gating() + if not bundle and "test" in self.task_factories: + indexed_tests = self._index_tests_by_resource(dbt_nodes, dbt_sources, dbt_unit_tests, task_keys) + gating = _Gating( + tests=indexed_tests, + ancestors=self._compute_ancestors(dbt_nodes, dbt_sources) if indexed_tests else {}, + resources_by_task_key={task_key: full_name for full_name, task_key in task_keys.items()}, + ) + + tasks = self._build_resource_tasks( + dbt_nodes, + bundle, + task_keys, + bundled_test_keys, + gating, + peers, + ) + + if bundle: + tasks.extend( + self._build_bundled_test_tasks( + dbt_nodes, + dbt_sources, + bundled_tests, + task_keys, + bundled_test_keys, + peers, + ) + ) + tasks.extend(self._build_standalone_test_tasks(standalone_tests, task_keys, peers)) + elif "test" in self.task_factories: + tasks.extend(self._build_unit_test_tasks(dbt_unit_tests, task_keys, peers)) + + return tasks + + @staticmethod + def _enabled_only(entries: dict) -> dict: + """ + Drops entries dbt has disabled. + + dbt normally files an `enabled=false` resource under the manifest's own `disabled` key, which + we never read — but not always: a versioned model whose declared version has no file leaves + its test in `nodes` with `config.enabled` false. dbt selects nothing for such a node, and + `dbt test` on a zero-match selector still exits 0, so a task built from it would go green + having asserted nothing. Filtering here rather than at each decision site keeps the task-key + map, the bundling classification and the dependency graph working from one view of the + manifest. Confirmed against dbt 1.12.0. + """ + return { + full_name: info + for full_name, info in entries.items() + if (info.get("config") or {}).get("enabled") is not False + } + + def _node_gets_own_task(self, full_name: str, node_info: dict, bundle: bool, standalone_test_ids: set[str]) -> bool: + """ + Whether a `dbt_nodes` entry becomes its own task (and so receives a task key). True for any + resource type with a factory, except single-resource test nodes in bundle mode — those fold + into their resource's bundled test task. The single authority for this decision, so the + task-key map and the task-building loops stay in agreement. + """ + resource_type = node_info["resource_type"] + if resource_type not in self.task_factories: + return False + if bundle and resource_type == "test" and full_name not in standalone_test_ids: + return False + return True + + def _emitted_unit_test_ids(self, dbt_unit_tests: dict, dbt_nodes: dict) -> list[str]: + """ + Full names of the unit tests that become their own task in per-test mode: those whose + target model resolves and is present in the manifest. This is the emission decision for + unit tests — the returned ids enter `task_ids`, so a unit test's presence in `task_keys` + is how every consumer knows it was emitted. + + Each versioned unit-test clone receives its own task and depends on its exact model version. + """ + emitted: list[str] = [] + for full_name, info in dbt_unit_tests.items(): + model_full_name = self._unit_test_model(info) + if model_full_name is not None and model_full_name in dbt_nodes: + emitted.append(full_name) + return emitted + + def _compute_ancestors(self, dbt_nodes: dict, dbt_sources: dict) -> dict[str, set[str]]: + """ + Maps each testable resource's full name to the set of resources it transitively depends + on (not including itself). Used in per-test mode to decide whether a test can safely + gate a downstream node: a test `T` with refs `R` is only safe to add to node `N`'s + deps if `R ⊆ ancestors(N)` — i.e. `N` already waits for all of `T`'s endpoints, + transitively. Otherwise adding `T` would create a cycle (since `T` depends on each + ref, and some ref might depend on `N`). + """ + resources = {**dbt_nodes, **dbt_sources} + dependencies: dict[str, set[str]] = {} + dependents: dict[str, list[str]] = {full_name: [] for full_name in resources} + for full_name, info in resources.items(): + direct_dependencies = { + dependency for dependency in info.get("depends_on", {}).get("nodes", []) if dependency in resources + } + dependencies[full_name] = direct_dependencies + for dependency in direct_dependencies: + dependents[dependency].append(full_name) + + unresolved_counts = { + full_name: len(direct_dependencies) for full_name, direct_dependencies in dependencies.items() + } + ready = [full_name for full_name, count in unresolved_counts.items() if count == 0] + heapq.heapify(ready) + ancestors: dict[str, set[str]] = {} + while ready: + full_name = heapq.heappop(ready) + node_ancestors: set[str] = set() + for dependency in sorted(dependencies[full_name]): + node_ancestors.add(dependency) + node_ancestors.update(ancestors[dependency]) + ancestors[full_name] = node_ancestors + + for dependent in sorted(dependents[full_name]): + unresolved_counts[dependent] -= 1 + if unresolved_counts[dependent] == 0: + heapq.heappush(ready, dependent) + + if len(ancestors) != len(resources): + unresolved = {full_name for full_name, count in unresolved_counts.items() if count > 0} + cycle = self._dependency_cycle(dependencies, unresolved) + raise ValueError( + f"Cannot compute test gates because the manifest contains the dependency cycle " + f"{' -> '.join(cycle)}. Regenerate the manifest after removing the cycle." + ) + return ancestors + + @staticmethod + def _dependency_cycle(dependencies: dict[str, set[str]], candidates: set[str]) -> list[str]: + """Returns one deterministic cycle from a graph that could not be topologically ordered.""" + active = 1 + complete = 2 + state: dict[str, int] = {} + for start in sorted(candidates): + if state.get(start) == complete: + continue + + path = [start] + positions = {start: 0} + state[start] = active + stack: list[tuple[str, Iterator[str]]] = [(start, iter(sorted(dependencies[start] & candidates)))] + while stack: + full_name, direct_dependencies = stack[-1] + try: + dependency = next(direct_dependencies) + except StopIteration: + stack.pop() + path.pop() + positions.pop(full_name) + state[full_name] = complete + continue + + dependency_state = state.get(dependency) + if dependency_state is None: + positions[dependency] = len(path) + path.append(dependency) + state[dependency] = active + stack.append((dependency, iter(sorted(dependencies[dependency] & candidates)))) + elif dependency_state == active: + return path[positions[dependency] :] + [dependency] + + raise RuntimeError("A non-topological dependency graph did not contain a cycle.") + + def _index_tests_by_resource( + self, dbt_nodes: dict, dbt_sources: dict, dbt_unit_tests: dict, task_keys: dict[str, str] + ) -> dict[str, list[tuple[str, frozenset[str]]]]: + """ + Maps each testable resource's full name to a list of (test_task_key, test_refs) pairs + for every emitted data test. A test's task exit status decides whether the dependency blocks: + warn-severity tests normally succeed, while `--warn-error` can make the same task fail. Keeping + one graph rule preserves dbt's ordering without reproducing its option-dependent severity logic. + + Unit tests are indexed too. A unit test has no severity — it always fails the run — so it + always gates. Only unit tests that were emitted as tasks (present in `task_keys`) are + indexed, so a downstream node never gates on a unit-test task that was skipped. + + The refs set is carried alongside each test so `_extend_deps_with_upstream_tests` can + avoid cycles: a test with refs that aren't all ancestors of a candidate node would + create a cycle if added as that node's dep. + """ + index: dict[str, list[tuple[str, frozenset[str]]]] = {} + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + if node_full_name in task_keys: + self._index_test(index, task_keys[node_full_name], node_info, dbt_nodes, dbt_sources) + + for unit_test_full_name, unit_test_info in dbt_unit_tests.items(): + if unit_test_full_name in task_keys: + self._index_test(index, task_keys[unit_test_full_name], unit_test_info, dbt_nodes, dbt_sources) + return index + + def _index_test( + self, + index: dict[str, list[tuple[str, frozenset[str]]]], + test_task_key: str, + test_info: dict, + dbt_nodes: dict, + dbt_sources: dict, + ) -> None: + """Indexes a test (data or unit) under each resource it references, carrying its ref set.""" + refs = self._testable_refs(test_info, dbt_nodes, dbt_sources) + for resource_full in refs: + index.setdefault(resource_full, []).append((test_task_key, refs)) + + def _testable_refs(self, test_info: dict, dbt_nodes: dict, dbt_sources: dict) -> frozenset[str]: + """Returns the models/seeds/snapshots/sources a test references, as present in the manifest.""" + refs: set[str] = set() + for dep in test_info.get("depends_on", {}).get("nodes", []): + if dep.startswith(self._DBT_TEST_TARGET_PREFIXES) and (dep in dbt_nodes or dep in dbt_sources): + refs.add(dep) + return frozenset(refs) + + @staticmethod + def _unit_test_model(unit_test_info: dict) -> str | None: + """ + Returns the full name of the model a unit test targets, or None if it can't be resolved. + + Prefers dbt's already-resolved `depends_on.nodes[0]`, which is the authority: dbt resolves + the `model` field to a node id at parse time and rewrites unit tests on *versioned* models + to target `model...v` while leaving `model` as the bare name. Rebuilding + `model..` from the `model` field would therefore miss versioned models + entirely (and dbt's ` ` spelling of the field), silently dropping the unit + test. Falls back to that reconstruction only when `depends_on` is absent, so manifests + that predate it keep working. + """ + for dep in unit_test_info.get("depends_on", {}).get("nodes", []): + if dep.startswith("model."): + return dep + model = unit_test_info.get("model") + package = unit_test_info.get("package_name") + if model and package: + return f"model.{package}.{model}" + return None + + @classmethod + def _extend_deps_with_upstream_tests( + cls, + node_full_name: str, + existing_deps: list[str] | None, + gating: _Gating, + ) -> list[str]: + """ + Adds safe test gates at the first downstream frontier. + + A test is eligible for node `N` when all of its refs are strict ancestors of `N`. Tests already + eligible at an immediate emitted dependency are inherited through that dependency and need not be + repeated. The remaining tests are the first frontier where the full ref set has become available. + """ + deps: list[str] = list(existing_deps or []) + eligible = cls._eligible_test_keys(node_full_name, gating) + inherited: set[str] = set() + for dependency_key in deps: + dependency = gating.resources_by_task_key.get(dependency_key) + if dependency is not None: + inherited.update(cls._eligible_test_keys(dependency, gating)) + deps.extend(sorted(eligible - inherited - set(deps))) + return deps + + @staticmethod + def _eligible_test_keys(node_full_name: str, gating: _Gating) -> frozenset[str]: + """Returns and caches tests whose complete ref set is among the node's strict ancestors.""" + cached = gating.eligible_test_keys.get(node_full_name) + if cached is not None: + return cached + + ancestors = gating.ancestors.get(node_full_name, set()) + eligible: set[str] = set() + for ancestor in ancestors: + for test_key, test_refs in gating.tests.get(ancestor, []): + if test_refs <= ancestors: + eligible.add(test_key) + result = frozenset(eligible) + gating.eligible_test_keys[node_full_name] = result + return result + + def _classify_tests( + self, dbt_nodes: dict, dbt_sources: dict, dbt_unit_tests: dict + ) -> tuple[dict[str, list[tuple[str, dict]]], list[tuple[str, dict]]]: + """ + Classifies test nodes for bundled mode so that no test is silently dropped. + + - Tests with exactly 1 testable dep: added by id to that resource's bundled + `_test` task. Each receives its own exact selection plan before the plans are + unioned by indirect-selection mode. + - Tests with >1 testable deps (cross-resource, e.g. `relationships`): emitted as their own + tasks with every available model, seed, or snapshot dependency. + - Tests with 0 testable deps (singular/custom tests that don't `ref()` or `source()` + any resource): also emitted as their own tasks, since no bundle would pick them up. + + Unit tests are assigned explicitly to their resolved model bundle. A model with only unit + tests therefore still gets a bundled task without relying on indirect selection to discover it. + + Returns: + (bundled_tests, standalone_tests): + - `bundled_tests`: exact data and unit test nodes grouped by the resource whose + bundled task runs them. + - `standalone_tests`: list of `(test_full_name, test_node_info)` for tests + that must run as individual tasks (cross-model or zero-dep). + """ + bundled_tests: dict[str, list[tuple[str, dict]]] = {} + standalone_tests: list[tuple[str, dict]] = [] + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + testable_deps = self._testable_refs(node_info, dbt_nodes, dbt_sources) + if len(testable_deps) == 1: + resource_id = next(iter(testable_deps)) + bundled_tests.setdefault(resource_id, []).append((node_full_name, node_info)) + else: + standalone_tests.append((node_full_name, node_info)) + + for unit_test_full_name, unit_test_info in dbt_unit_tests.items(): + model_full_name = self._unit_test_model(unit_test_info) + if model_full_name is not None and model_full_name in dbt_nodes: + bundled_tests.setdefault(model_full_name, []).append((unit_test_full_name, unit_test_info)) + return bundled_tests, standalone_tests + + def _build_resource_tasks( + self, + dbt_nodes: dict, + bundle: bool, + task_keys: dict[str, str], + bundled_test_keys: dict[str, str], + gating: _Gating, + peers: dict, + ) -> list[DbtTask]: + """Builds tasks for every non-test resource, plus per-test tasks when not bundling.""" + # A tested resource resolves to its bundle; sources gain a scheduled key only through this map. + dependency_task_keys = {**task_keys, **bundled_test_keys} if bundle else task_keys + tasks: list[DbtTask] = [] + for node_full_name, node_info in dbt_nodes.items(): + if node_full_name not in task_keys: + continue + if bundle and node_info["resource_type"] == "test": + # Standalone tests are keyed but built by `_build_standalone_test_tasks`, not here. + continue + + resource_type = node_info["resource_type"] + task_key = task_keys[node_full_name] + factory = self.task_factories[resource_type] + if resource_type == "test": + plan = self._test_selection_plan(node_info, peers) + task = cast(TestTaskFactory, factory).create_task( + plan.select, + node_info["name"], + node_info, + task_key, + task_keys, + indirect_selection=plan.indirect_selection, + ) + else: + task = factory.create_task( + self._node_select(node_info, peers=peers), + node_info["name"], + node_info, + task_key, + dependency_task_keys, + ) + + if resource_type in self._GATEABLE_TYPES and not bundle: + task = self._gate_task(task, node_full_name, gating) + tasks.append(task) + return tasks + + def _gate_task( + self, + task: DbtTask, + node_full_name: str, + gating: _Gating, + ) -> DbtTask: + """Adds per-test-mode gates at the first downstream frontier.""" + if not gating.tests: + return task + deps = self._extend_deps_with_upstream_tests(node_full_name, task.depends_on, gating) + return replace(task, depends_on=deps) + + def _build_bundled_test_tasks( + self, + dbt_nodes: dict, + dbt_sources: dict, + bundled_tests: dict[str, list[tuple[str, dict]]], + task_keys: dict[str, str], + bundled_test_keys: dict[str, str], + peers: dict, + ) -> list[DbtTask]: + """Emits one task per tested resource from unions of exact per-test selection plans.""" + test_factory = cast(TestTaskFactory, self.task_factories["test"]) + tasks: list[DbtTask] = [] + for full_name, tests in sorted(bundled_tests.items()): + is_source = full_name.startswith("source.") + info = dbt_sources[full_name] if is_source else dbt_nodes[full_name] + # A source never gets a task of its own, so it has no gate. Any other parent must, or the + # bundle would run its tests with nothing ordering them after the resource was built. Raise + # rather than emit the ungated task: `resource_type` having no registered factory is a + # supported library configuration (see `_node_gets_own_task`), and a `KeyError` here would + # escape `main`'s handler as a traceback that names no resource. + if not is_source and full_name not in task_keys: + raise ValueError( + f"Cannot bundle the tests of {full_name!r}: no task was generated for it, so the " + f"bundled test task would run unordered against it. Register a task factory for " + f"{info.get('resource_type')!r} resources, or generate one task per test instead of " + f"bundling." + ) + tasks.append( + test_factory.create_bundled_task( + task_key=bundled_test_keys[full_name], + selects_by_indirect_selection=self._bundled_selects_by_mode(tests, peers), + deps_command_name=info["name"], + depends_on=[] if is_source else [task_keys[full_name]], + ) + ) + return tasks + + @classmethod + def _bundled_selects_by_mode(cls, tests: list[tuple[str, dict]], peers: dict) -> dict[str, list[str]]: + """Builds and validates the deterministic selector groups for one test bundle.""" + selects_by_mode: dict[str, list[str]] = {} + test_info_by_mode: dict[str, dict] = {} + for _, test_info in sorted(tests, key=lambda item: item[0]): + plan = cls._test_selection_plan(test_info, peers) + selects_by_mode.setdefault(plan.indirect_selection, []).append(plan.select) + test_info_by_mode.setdefault(plan.indirect_selection, test_info) + for mode, selects in selects_by_mode.items(): + cls._assert_no_dynamic_reference(" ".join(sorted(selects)), test_info_by_mode[mode]) + return selects_by_mode + + def _build_standalone_test_tasks( + self, + standalone_tests: list[tuple[str, dict]], + task_keys: dict[str, str], + peers: dict, + ) -> list[DbtTask]: + """ + Emits one task per standalone test — cross-resource tests (e.g. `relationships`) gated on + every referenced model, seed, or snapshot task, plus zero-dep singular tests that bundles + cannot cover. + """ + test_factory = cast(TestTaskFactory, self.task_factories["test"]) + tasks: list[DbtTask] = [] + for test_full_name, test_info in sorted(standalone_tests, key=lambda item: item[0]): + test_task_key = task_keys[test_full_name] + plan = self._test_selection_plan(test_info, peers) + tasks.append( + test_factory.create_task( + plan.select, + test_info["name"], + test_info, + test_task_key, + task_keys, + indirect_selection=plan.indirect_selection, + ) + ) + return tasks + + def _build_unit_test_tasks( + self, + dbt_unit_tests: dict, + task_keys: dict[str, str], + peers: dict, + ) -> list[DbtTask]: + """ + Emits one task per unit test, selected by its full FQN and gated on the model it tests. + Only unit tests that received a task key (see `_emitted_unit_test_ids`) are emitted; unit + tests whose target model is absent from the manifest were never keyed and are skipped, so + their task can't gate on a model task that is never created. Used in per-test mode; in + bundled mode unit tests are assigned explicitly to their model's exact-selector bundle. + + Versioned unit-test clones receive independent parent-scoped selection plans, so each task waits + only for and runs only against its exact model version. + """ + test_factory = cast(TestTaskFactory, self.task_factories["test"]) + + tasks: list[DbtTask] = [] + for unit_test_full_name, unit_test_info in sorted(dbt_unit_tests.items()): + if unit_test_full_name not in task_keys: + continue + plan = self._test_selection_plan(unit_test_info, peers) + tasks.append( + test_factory.create_task( + plan.select, + unit_test_info["name"], + unit_test_info, + task_keys[unit_test_full_name], + task_keys, + indirect_selection=plan.indirect_selection, + ) + ) + return tasks + + +class _SelectorIndex(dict): + """ + The manifest's selectable resources, plus indexes for narrowing an exactness check. + + `_assert_exact` runs once per node, so evaluating each selector against every node makes generation + quadratic — measured at 90 seconds for a 6,000-node manifest, and real projects are larger. Each + index maps a term value to the nodes it *could* match, so a check scans a handful of candidates + instead of the manifest. + + Every bucket is a sound **superset** of the true matches: `_matching_ids` still evaluates the full + predicate on whatever comes back, so narrowing can only cost time, never change the answer. + `test_selector_index_narrowing_matches_a_full_scan` pins that equivalence against a full scan. + + Subclasses `dict` so it *is* the peers mapping: callers that only iterate or look up by id need not + know the indexes exist. + """ + + def __init__(self, peers: dict): + super().__init__(peers) + self._by_package: dict[str, dict] = {} + self._by_file: dict[str, dict] = {} + self._by_test_name: dict[str, dict] = {} + self._by_fqn_term: dict[str, dict] = {} + self._by_version_suffix: dict[str, dict] = {} + self._tests_by_parent: dict[str, set[str]] = {} + for full_name, info in peers.items(): + self._add(full_name, info) + self._index_test_parents(full_name, info, peers) + + def _index_test_parents(self, full_name: str, info: dict, peers: dict) -> None: + """Indexes a test under each enabled testable parent.""" + if info.get("resource_type") not in {"test", "unit_test"}: + return + for parent in info.get("depends_on", {}).get("nodes", []): + if parent.startswith(_DBT_TEST_TARGET_PREFIXES) and parent in peers: + self._tests_by_parent.setdefault(parent, set()).add(full_name) + + def _add(self, full_name: str, info: dict) -> None: + """Files one node under every key a selector term could reach it by.""" + self._by_package.setdefault(info.get("package_name") or "", {})[full_name] = info + # A path containing a backslash has both POSIX and Windows interpretations. Index every possible + # base name and stem so narrowing cannot hide a collision under either runtime path flavour. + for base in _candidate_file_names(info.get("original_file_path") or ""): + for key in dict.fromkeys((base, base.rsplit(".", 1)[0] if "." in base else base)): + self._by_file.setdefault(key, {})[full_name] = info + test_name = (info.get("test_metadata") or {}).get("name") or "" + if test_name: + self._by_test_name.setdefault(test_name, {})[full_name] = info + for term in self._fqn_terms(info): + self._by_fqn_term.setdefault(term, {})[full_name] = info + if suffix := self._version_suffix(info): + self._by_version_suffix.setdefault(suffix, {})[full_name] = info + + def tests_attached_to_any(self, parent_ids: set[str]) -> set[str]: + """Returns indexed tests having at least one parent in `parent_ids`.""" + tests: set[str] = set() + for parent_id in parent_ids: + tests.update(self._tests_by_parent.get(parent_id, ())) + return tests + + @staticmethod + def _fqn_terms(info: dict) -> set[str]: + """ + Every finite fqn term that directly identifies this node under dbt's matching rules. + + The positional matcher accepts every prefix of the flattened fqn, both with and without the + package. The leaf shortcut additionally accepts the raw leaf (or the model name for a versioned + model), which can differ from a flattened prefix when it contains a dot. Version suffix matching + is indexed separately because dbt ignores any earlier selector components in that shortcut. + """ + fqn = info.get("fqn") or [] + if not fqn: + # A truncated hand-written manifest can omit fqn; the matcher then falls back to the name. + name = info.get("name") or "" + return {name} if name else set() + terms: set[str] = set() + for candidate in (fqn, fqn[1:]): + flat = _flatten_fqn(candidate) + terms.update(".".join(flat[:length]) for length in range(1, len(flat) + 1)) + if info.get("resource_type") == "model" and info.get("version") is not None and len(fqn) >= 2: + terms.add(fqn[-2]) + else: + terms.add(fqn[-1]) + return terms + + @staticmethod + def _version_suffix(info: dict) -> str: + """The suffix used by dbt's versioned-model leaf shortcut, if this node has one.""" + fqn = info.get("fqn") or [] + if info.get("resource_type") == "model" and info.get("version") is not None and len(fqn) >= 2: + return "_".join(fqn[-2:]) + return "" + + def narrow(self, terms: list[str]) -> dict: + """ + The candidate set for `terms` — a superset of the true matches, so the caller still decides. + + Returns the smallest single bucket rather than intersecting across terms: the intersection is + also sound (terms are ANDed) but measured slower, because building a set from a bucket the size + of the manifest costs more than the extra predicate evaluations it saves. + """ + smallest: dict | None = None + for term in terms: + method, _, value = term.partition(":") + if not _ or method == "fqn": + bucket = self._fqn_candidates(value if method == "fqn" else term) + elif method == "package": + bucket = self._by_package.get(value, {}) + elif method == "file": + bucket = self._by_file.get(value, {}) + elif method == "test_name": + bucket = self._by_test_name.get(value, {}) + else: + continue + if smallest is None or len(bucket) < len(smallest): + smallest = bucket + return self if smallest is None else smallest + + def _fqn_candidates(self, term: str) -> dict: + """ + The nodes an fqn `term` could match through a positional, leaf, or version-suffix rule. + + Prefixes are indexed as complete terms rather than by their first segment, keeping selectors + efficient when an ambiguous path makes `file:` unavailable. The version shortcut compares only + the selector's last two components, so its separate bucket is unioned here. + """ + candidates = dict(self._by_fqn_term.get(term, {})) + suffix = "_".join(term.split(".")[-2:]) + candidates.update(self._by_version_suffix.get(suffix, {})) + return candidates diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/dbt_task.py b/contrib/dbt_factory/src/databricks_dbt_factory/dbt_task.py new file mode 100644 index 00000000..36c2d70e --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/dbt_task.py @@ -0,0 +1,173 @@ +import json +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +from databricks_dbt_factory.utils import DYNAMIC_VALUE_REFERENCE + + +_MAX_NOTEBOOK_BASE_PARAMETERS_BYTES = 1_000_000 + + +class TaskType(Enum): + """Supported task types for generated Databricks tasks.""" + + DBT = "dbt" + NOTEBOOK = "notebook" + + +@dataclass(frozen=True) +class DbtTaskOptions: + environment_key: str = "Default" # serverless env + """The key of an environment. It has to be unique within a job.""" + + catalog: str | None = None + """Optional name of the catalog to use. The value is the top level in the 3-level namespace of + Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a + warehouse_id is specified. Requires dbt-databricks >= 1.1.1.""" + + profiles_directory: str | None = None + """Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is + specified. If no warehouse_id is specified and this folder is unset, the root directory is used.""" + + project_directory: str | None = None + """Path to the project directory. Optional for Git sourced tasks, in which case if no value is + provided, the root of the Git repository is used.""" + + schema: str | None = None + """Optional schema to write to. This parameter is only used when a warehouse_id is also provided. + If not provided, the `default` schema is used.""" + + source: str | None = None + """Optional location type of the project directory. When set to `WORKSPACE`, the project will be + retrieved from the local Databricks workspace. When set to `GIT`, the project will be retrieved + from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if + `git_source` is defined and `WORKSPACE` otherwise. + + * `WORKSPACE`: Project is located in Databricks workspace. * `GIT`: Project is located in cloud + Git provider.""" + + warehouse_id: str | None = None + """ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the + profile and connection details to dbt. It can be overridden on a per-command basis by using the + `--profiles-dir` command line argument.""" + + dbt_deps_enabled: bool = False + """Optional flag to enable dbt deps to be run before each task. Defaults to False.""" + + dbt_tasks_deps: list[str] = field(default_factory=list) + """Optional comma separated list of tasks that requires dbt debs. Only in effect if dbt_deps_enabled is enabled.""" + + task_type: TaskType = TaskType.NOTEBOOK + """Task type to generate: `TaskType.NOTEBOOK` for a notebook_task wrapper (default), + `TaskType.DBT` for a native dbt_task. Strings are accepted and coerced — any value outside the + enum raises `ValueError`. Notebook mode enables base environment support and environment + variables on serverless.""" + + notebook_path: str | None = None + """Path to the dbt runner notebook. Required when task_type is `TaskType.NOTEBOOK`.""" + + job_cluster_key: str | None = None + """Job cluster key for running tasks on job compute instead of serverless.""" + + def __post_init__(self): + if not isinstance(self.task_type, TaskType): + object.__setattr__(self, "task_type", TaskType(self.task_type)) + if self.task_type is TaskType.NOTEBOOK: + if not self.notebook_path: + raise ValueError("notebook_path is required when task_type=NOTEBOOK.") + unsupported = [] + for name, value in ( + ("warehouse_id", self.warehouse_id), + ("schema", self.schema), + ("catalog", self.catalog), + ): + if value: + unsupported.append(name) + if unsupported: + raise ValueError( + f"{', '.join(unsupported)} cannot be set with task_type=NOTEBOOK; " + "notebook tasks connect via profiles.yml." + ) + + +@dataclass(frozen=True) +class DbtTask: + """Represents a dbt task in the Databricks job definition.""" + + task_key: str + commands: list[str] + options: DbtTaskOptions + depends_on: list[str] | None = None + + def to_dict(self) -> dict: + """Converts the Task to a dictionary suitable for the job definition.""" + if self.options.task_type is TaskType.NOTEBOOK: + return self._to_notebook_dict() + return self._to_dbt_dict() + + def _base_spec(self) -> dict[str, Any]: + spec: dict[str, Any] = { + "task_key": self.task_key, + "depends_on": [{"task_key": dep} for dep in (self.depends_on or [])], + } + if self.options.job_cluster_key: + spec["job_cluster_key"] = self.options.job_cluster_key + else: + spec["environment_key"] = self.options.environment_key + return spec + + def _to_dbt_dict(self) -> dict[str, Any]: + spec = self._base_spec() + dbt_task: dict[str, Any] = {"commands": self.commands} + + if self.options.source: + dbt_task["source"] = self.options.source + if self.options.project_directory: + dbt_task["project_directory"] = self.options.project_directory + if self.options.schema: + dbt_task["schema"] = self.options.schema + if self.options.warehouse_id: + dbt_task["warehouse_id"] = self.options.warehouse_id + if self.options.catalog: + dbt_task["catalog"] = self.options.catalog + if self.options.profiles_directory: + dbt_task["profiles_directory"] = self.options.profiles_directory + + spec["dbt_task"] = dbt_task + return spec + + def _to_notebook_dict(self) -> dict[str, Any]: + serialized_commands = json.dumps(self.commands) + if DYNAMIC_VALUE_REFERENCE.search(serialized_commands): + raise ValueError( + f"Notebook task {self.task_key!r} serialized dbt_commands forms a Databricks dynamic value " + f"reference; rename the selected resources or change the dbt command options." + ) + base_parameters: dict[str, str] = { + "dbt_commands": serialized_commands, + } + if self.options.project_directory: + base_parameters["project_directory"] = self.options.project_directory + if self.options.profiles_directory: + base_parameters["profiles_directory"] = self.options.profiles_directory + + serialized_size = len(json.dumps(base_parameters, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + if serialized_size > _MAX_NOTEBOOK_BASE_PARAMETERS_BYTES: + raise ValueError( + f"Notebook task {self.task_key!r} serializes base_parameters to {serialized_size:,} bytes; " + f"Databricks allows at most {_MAX_NOTEBOOK_BASE_PARAMETERS_BYTES:,} bytes. Shorten its dbt " + f"commands or path options, reduce the tests in this bundle, or generate native dbt tasks " + f"with --task-type dbt." + ) + + notebook_task: dict[str, Any] = { + "notebook_path": self.options.notebook_path, + "base_parameters": base_parameters, + } + if self.options.source: + notebook_task["source"] = self.options.source + + spec = self._base_spec() + spec["notebook_task"] = notebook_task + return spec diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/notebook/run_dbt_command.py b/contrib/dbt_factory/src/databricks_dbt_factory/notebook/run_dbt_command.py new file mode 100644 index 00000000..accb8f6a --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/notebook/run_dbt_command.py @@ -0,0 +1,114 @@ +# Databricks notebook source + +import json +import os +import shlex +import shutil +import tempfile +from urllib.parse import urlparse + +from databricks.sdk import WorkspaceClient +from dbt.cli.main import dbtRunner + +# COMMAND ---------- + +dbutils.widgets.text("dbt_commands", "") +dbutils.widgets.text("project_directory", "") +dbutils.widgets.text("profiles_directory", "") + +dbt_commands = dbutils.widgets.get("dbt_commands") +project_directory = dbutils.widgets.get("project_directory") +profiles_directory = dbutils.widgets.get("profiles_directory") + +if not dbt_commands: + raise ValueError("dbt_commands parameter is required") + +# COMMAND ---------- + +# Authenticate dbt with the notebook's own credentials via the Databricks SDK. The captured token +# is short-lived (about an hour) and does not self-refresh, so a single task running longer than its +# lifetime can fail mid-run. +ws = WorkspaceClient() +os.environ["DBT_ACCESS_TOKEN"] = ws.config.authenticate()["Authorization"].removeprefix("Bearer ").strip() + +# dbt's host must be a bare hostname, but `config.host` is a full URL (scheme + host, and potentially +# a trailing slash or path). Parse it and keep only the netloc so a value like +# "https://my-workspace.databricks.com/" yields "my-workspace.databricks.com". +_parsed = urlparse(ws.config.host) +os.environ["DBT_HOST"] = _parsed.netloc or _parsed.path.strip("/") + +# chdir to the dbt project so dbt runs from inside it. Relative `project_directory` is +# resolved against this notebook's own workspace location — the same anchor native +# `dbt_task` uses. Auto-copy mode sends `.` (resolves to the notebook's dir, which is +# project root by construction); user-pinned `--notebook-path` with relative +# `--project-directory` resolves against wherever the user placed the notebook; absolute +# `project_directory` is used as-is. +if project_directory: + ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext() + notebook_dir = os.path.dirname("/Workspace" + ctx.notebookPath().get()) + target_dir = ( + project_directory + if os.path.isabs(project_directory) + else os.path.normpath(os.path.join(notebook_dir, project_directory)) + ) + os.chdir(target_dir) + +# Point dbt's artifact and log output at a private per-task dir so the parallel tasks of a job +# don't contend on the shared workspace `target/`/`logs/`. +local_dir = tempfile.mkdtemp(prefix="dbt_local_") +os.environ["DBT_TARGET_PATH"] = local_dir +os.environ["DBT_LOG_PATH"] = local_dir + + +# If a pre-built msgpack sits next to the project, deserialize it into a manifest and inject it into +# dbtRunner to skip dbt's parse phase (re-reading/hashing every file + DAG rebuild) on each task. +# Falls back to a normal parse if the msgpack is absent or unusable. +manifest = None +prebuilt_manifest_path = os.path.join("target", "partial_parse.msgpack") +if os.path.exists(prebuilt_manifest_path): + try: + from dbt.contracts.graph.manifest import Manifest + + with open(prebuilt_manifest_path, "rb") as f: + manifest = Manifest.from_msgpack(f.read()) + manifest.build_flat_graph() + print(f"[dbt-factory] injecting pre-built manifest from {prebuilt_manifest_path} (skipping dbt parse)") + except Exception as e: + print(f"[dbt-factory] manifest injection unavailable, falling back to dbt parse: {e}") + manifest = None + +try: + runner = dbtRunner(manifest=manifest) + + for command_str in json.loads(dbt_commands): + command_str = command_str.strip() + if not command_str: + continue + + if command_str.startswith("dbt "): + command_str = command_str[4:] + + args = shlex.split(command_str) + + if profiles_directory: + args.extend(["--profiles-dir", profiles_directory]) + + print(f"Running: dbt {' '.join(args)}") + print("-" * 60) + + result = runner.invoke(args) + + if not result.success: + detail = result.exception or result.result or "(no further details)" + raise RuntimeError(f"dbt command failed: dbt {' '.join(args)}\n{detail}") + + print(f"Completed successfully: dbt {' '.join(args)}") +finally: + os.environ.pop("DBT_ACCESS_TOKEN", None) + os.environ.pop("DBT_HOST", None) + os.environ.pop("DBT_TARGET_PATH", None) + os.environ.pop("DBT_LOG_PATH", None) + # Remove the private per-task target/log dir; on reused (all-purpose) clusters these + # would otherwise accumulate under the system temp dir for the life of the cluster. + if local_dir: + shutil.rmtree(local_dir, ignore_errors=True) diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/task_factory.py b/contrib/dbt_factory/src/databricks_dbt_factory/task_factory.py new file mode 100644 index 00000000..fe3d9b2b --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/task_factory.py @@ -0,0 +1,430 @@ +import shlex +from abc import ABC, abstractmethod +from collections.abc import Sequence + +from databricks_dbt_factory.dbt_task import DbtTask, DbtTaskOptions +from databricks_dbt_factory.utils import DYNAMIC_VALUE_REFERENCE + + +_RESERVED_DBT_SELECTION_OPTIONS = frozenset( + { + "--select", + "--models", + "--model", + "--exclude", + "--selector", + "--resource-type", + "--resource-types", + "--exclude-resource-type", + "--exclude-resource-types", + } +) +_RESERVED_DBT_PARSE_CONTEXT_OPTIONS = frozenset({"--vars", "--profile", "--profiles-dir", "--project-dir"}) +_RESERVED_DBT_SHORT_SELECTION_OPTIONS = frozenset({"-s", "-m"}) +_RESERVED_DBT_SHORT_TARGET_OPTIONS = frozenset({"-t"}) +_DBT_SHORT_OPTIONS_WITH_ATTACHED_VALUES = frozenset({"-r", "-t"}) +_DBT_TARGET_OPTIONS_WITH_SEPARATE_VALUES = frozenset({"--target", "-t"}) +_DBT_TARGET_VALUE_ERROR = "dbt target requires a nonempty value." +_EXTRA_DBT_PARSE_CONTEXT_REMEDIES = { + "--target": "--target", + "-t": "--target", + "--profiles-dir": "--profiles-directory", + "--project-dir": "--project-directory", +} + + +def _reserved_short_option(token: str, reserved_options: frozenset[str]) -> str | None: + """Returns a reserved option embedded in one dbt short-option token.""" + if not token.startswith("-") or token.startswith("--"): + return None + + for character in token[1:]: + option = f"-{character}" + if option in reserved_options: + return option + if option in _DBT_SHORT_OPTIONS_WITH_ATTACHED_VALUES: + break + return None + + +def _reserved_parse_context_option(token: str) -> str | None: + """Returns a parse-context option that is unsafe at every validation boundary.""" + long_option = token.partition("=")[0] + if long_option in _RESERVED_DBT_PARSE_CONTEXT_OPTIONS: + return long_option + return None + + +def _target_option(token: str) -> str | None: + """Returns the target option represented by one token, including its short form.""" + long_option = token.partition("=")[0] + if long_option == "--target": + return long_option + return _reserved_short_option(token, _RESERVED_DBT_SHORT_TARGET_OPTIONS) + + +def _reserved_selection_option(token: str) -> str | None: + """Returns a selection option that conflicts with factory-owned selection.""" + if token == "--": + return token + long_option = token.partition("=")[0] + if long_option in _RESERVED_DBT_SELECTION_OPTIONS: + return long_option + return _reserved_short_option(token, _RESERVED_DBT_SHORT_SELECTION_OPTIONS) + + +def _is_supported_leading_target(token: str) -> bool: + """Returns whether a token is one of the controlled leading target forms.""" + return ( + token in _DBT_TARGET_OPTIONS_WITH_SEPARATE_VALUES + or token.startswith("--target=") + or (token.startswith("-t") and not token.startswith("--") and len(token) > 2) + ) + + +def _raise_parse_context_error(option: str, reject_target: bool) -> None: + """Raises the boundary-specific error for a runtime parse-context override.""" + if reject_target: + dedicated_option = _EXTRA_DBT_PARSE_CONTEXT_REMEDIES.get(option) + remedy = f" Use the dedicated {dedicated_option} argument instead." if dedicated_option else "" + raise ValueError( + f"--extra-dbt-command-options cannot include parse-context option {option!r}.{remedy} " + "The runtime parse context must match the supplied manifest." + ) + raise ValueError( + f"dbt command options cannot include parse-context option {option!r}; " + "the runtime parse context must match the supplied manifest." + ) + + +def _validate_dbt_option_token(token: str, token_index: int, reject_target: bool) -> bool: + """Validates one option token and reports whether its next token is a target value.""" + reserved_option = _reserved_selection_option(token) + if reserved_option is not None: + raise ValueError( + f"dbt command options cannot include selection option {reserved_option!r}; " + f"the factory owns selection so each task addresses only its intended resource." + ) + + target_option = _target_option(token) + if target_option is not None: + if reject_target: + _raise_parse_context_error(target_option, reject_target=True) + if token_index != 0 or not _is_supported_leading_target(token): + raise ValueError( + "dbt command options can include at most one target, only as the leading " + f"factory-controlled option; found {target_option!r} elsewhere." + ) + if token == "--target=": + raise ValueError(_DBT_TARGET_VALUE_ERROR) + return token in _DBT_TARGET_OPTIONS_WITH_SEPARATE_VALUES + + parse_context_option = _reserved_parse_context_option(token) + if parse_context_option is not None: + _raise_parse_context_error(parse_context_option, reject_target) + return False + + +def _validate_dbt_options(dbt_options: str, reject_target: bool) -> None: + """Rejects options that can invalidate the factory's manifest-derived task guarantees.""" + if DYNAMIC_VALUE_REFERENCE.search(dbt_options): + raise ValueError( + "dbt command options cannot contain a Databricks dynamic value reference; " + "its runtime value could change the resource selection owned by the factory." + ) + try: + tokens = shlex.split(dbt_options) + except ValueError as error: + raise ValueError(f"Cannot parse dbt command options: {error}.") from error + + option_value_expected = False + for token_index, token in enumerate(tokens): + if option_value_expected: + if not token: + raise ValueError(_DBT_TARGET_VALUE_ERROR) + option_value_expected = False + continue + option_value_expected = _validate_dbt_option_token(token, token_index, reject_target) + if option_value_expected: + raise ValueError(_DBT_TARGET_VALUE_ERROR) + + +def validate_dbt_options(dbt_options: str) -> None: + """Validates options supplied to a task factory, where its leading target remains allowed.""" + _validate_dbt_options(dbt_options, reject_target=False) + + +def validate_extra_dbt_options(dbt_options: str) -> None: + """Validates CLI extra options, including parse context managed by dedicated arguments.""" + _validate_dbt_options(dbt_options, reject_target=True) + + +class DbtDependencyResolver: + @staticmethod + def resolve(node_info: dict, task_keys: dict[str, str]) -> list[str]: + """ + Resolves every scheduled direct dbt dependency to its Databricks task key. + + Args: + node_info (dict): The dbt manifest entry for the node. + task_keys (dict[str, str]): Effective task key per scheduled dbt dependency. + + Returns: + list[str]: Resolved upstream task keys. + """ + deps = node_info.get("depends_on", {}).get("nodes", []) + resolved_deps = [] + for node_full_name in deps: + task_key = task_keys.get(node_full_name) + if task_key is not None: + resolved_deps.append(task_key) + return resolved_deps + + +class TaskFactory(ABC): + """Abstract base class for creating tasks.""" + + def __init__(self, resolver: DbtDependencyResolver, task_options: DbtTaskOptions, dbt_options: str = ""): + """ + Initializes the TaskFactory. + + Args: + resolver (DbtDependencyResolver): An instance of DbtDependencyResolver to resolve dependencies. + task_options (DbtTaskOptions): Options for the task. + dbt_options (str, optional): Additional DBT options. Defaults to "". + """ + self.resolver = resolver + self.task_options = task_options + self.dbt_options = dbt_options + + @property + def dbt_options(self) -> str: + """The validated dbt options appended to every command emitted by this factory.""" + return self._dbt_options + + @dbt_options.setter + def dbt_options(self, value: str) -> None: + validate_dbt_options(value) + self._dbt_options = value + + @abstractmethod + def create_task( + self, select: str, deps_command_name: str, dbt_node_info: dict, task_key: str, task_keys: dict[str, str] + ) -> DbtTask: + """ + Abstract method to create a task. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used by `get_dbt_deps_command` to decide whether + to prepend `dbt deps` (matched against `--dbt-tasks-deps`). + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + + Returns: + DbtTask: An instance of Task. + """ + + def get_dbt_deps_command(self, dbt_task_name: str) -> str | None: + """Adds the dbt deps command if enabled and applicable. + Only return the command if enabled, and available in the task deps or not specific tasks provided. + + Args: + dbt_task_name (str): Name of the DBT task. + """ + if self.task_options.dbt_deps_enabled and ( + not self.task_options.dbt_tasks_deps or dbt_task_name in self.task_options.dbt_tasks_deps + ): + return self._build_dbt_command("deps") + return None + + def _build_dbt_command( + self, + subcommand: str, + *, + select: str | Sequence[str] | None = None, + indirect_selection: str | None = None, + ) -> str: + """ + Builds and validates one complete dbt command. + + Several selectors are passed as repeated `--select` arguments rather than joined into one, so + the union does not depend on the consumer preserving shell quoting. dbt unions repeated + occurrences; confirmed on dbt 1.12.0 under both `empty` and `cautious` indirect selection. + """ + parts = ["dbt", subcommand] + if select is not None: + selectors = [select] if isinstance(select, str) else list(select) + for selector in selectors: + parts.extend(("--select", shlex.quote(selector))) + if self.dbt_options: + parts.append(self.dbt_options) + if indirect_selection is not None: + parts.extend(("--indirect-selection", shlex.quote(indirect_selection))) + + command = " ".join(parts) + if DYNAMIC_VALUE_REFERENCE.search(command): + raise ValueError( + "The final dbt command contains a Databricks dynamic value reference; rename the " + "selected resource or change the dbt command options so no `{{...}}` reference is formed." + ) + return command + + +class ModelTaskFactory(TaskFactory): + """Factory for creating model tasks.""" + + def create_task( + self, select: str, deps_command_name: str, dbt_node_info: dict, task_key: str, task_keys: dict[str, str] + ) -> DbtTask: + """ + Creates a model task. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used to decide whether to prepend `dbt deps`. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + + Returns: + DbtTask: An instance of Task. + """ + depends_on = self.resolver.resolve(dbt_node_info, task_keys) + + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(self._build_dbt_command("run", select=select)) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class SnapshotTaskFactory(TaskFactory): + """Factory for creating snapshot tasks.""" + + def create_task( + self, select: str, deps_command_name: str, dbt_node_info: dict, task_key: str, task_keys: dict[str, str] + ) -> DbtTask: + """ + Creates a snapshot task. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used to decide whether to prepend `dbt deps`. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + + Returns: + DbtTask: An instance of Task. + """ + depends_on = self.resolver.resolve(dbt_node_info, task_keys) + + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(self._build_dbt_command("snapshot", select=select)) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class SeedTaskFactory(TaskFactory): + """Factory for creating seed tasks.""" + + def create_task( + self, select: str, deps_command_name: str, dbt_node_info: dict, task_key: str, task_keys: dict[str, str] + ) -> DbtTask: + """ + Creates a seed task. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used to decide whether to prepend `dbt deps`. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + + Returns: + DbtTask: An instance of Task. + """ + depends_on = self.resolver.resolve(dbt_node_info, task_keys) + + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(self._build_dbt_command("seed", select=select)) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class TestTaskFactory(TaskFactory): + """Factory for creating test tasks.""" + + def create_task( + self, + select: str, + deps_command_name: str, + dbt_node_info: dict, + task_key: str, + task_keys: dict[str, str], + indirect_selection: str = "empty", + ) -> DbtTask: + """ + Creates a test task for a single dbt test node. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used to decide whether to prepend `dbt deps`. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + indirect_selection (str): dbt indirect-selection mode required by the selection plan. + + Returns: + DbtTask: An instance of Task. + """ + depends_on = self.resolver.resolve(dbt_node_info, task_keys) + + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + # The plan's mode is appended after user options so the command cannot override the selector's + # correctness contract. Direct test selectors use `empty`; parent-scoped selectors use `cautious`. + commands.append(self._build_dbt_command("test", select=select, indirect_selection=indirect_selection)) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + def create_bundled_task( + self, + task_key: str, + selects_by_indirect_selection: dict[str, list[str]], + deps_command_name: str, + depends_on: list[str], + ) -> DbtTask: + """ + Creates one Databricks task that runs an exact set of single-resource tests. + + Test selectors with the same indirect-selection mode are joined as one dbt union. Keeping + modes separate preserves each selector plan's exactness while limiting a bundle to at most + two test commands (`empty` and `cautious`). + + Args: + task_key (str): Key for the bundled task. + selects_by_indirect_selection (dict[str, list[str]]): Exact test selectors grouped by + the indirect-selection mode required by their selection plans. + deps_command_name (str): Name used by `get_dbt_deps_command` to decide whether to prepend `dbt deps`. + depends_on (list[str]): Upstream task keys this bundled task should gate on. + + Returns: + DbtTask: An instance of Task. + """ + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + unsupported_modes = set(selects_by_indirect_selection) - {"empty", "cautious"} + if unsupported_modes: + raise ValueError(f"Unsupported dbt indirect-selection modes: {', '.join(sorted(unsupported_modes))}.") + for indirect_selection in ("empty", "cautious"): + if indirect_selection not in selects_by_indirect_selection: + continue + selects = sorted(selects_by_indirect_selection[indirect_selection]) + commands.append(self._build_dbt_command("test", select=selects, indirect_selection=indirect_selection)) + + return DbtTask(task_key, commands, self.task_options, depends_on) diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/utils.py b/contrib/dbt_factory/src/databricks_dbt_factory/utils.py new file mode 100644 index 00000000..02ff51a2 --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/utils.py @@ -0,0 +1,255 @@ +import hashlib +import json +import re +from collections.abc import Iterable + +# Databricks substitutes complete dynamic value references in task string fields before execution. +DYNAMIC_VALUE_REFERENCE = re.compile(r"\{\{[^{}]+\}\}") + +# Databricks caps task keys at 100 characters (letters, numbers, underscores, hyphens). +MAX_TASK_KEY_LENGTH = 100 + +# dbt resource types whose task key is `_` (the type doubles as the suffix). +_SUFFIXED_TYPES = frozenset({"model", "seed", "snapshot"}) + + +def _resource_name(unique_id: str) -> str: + """ + The dbt resource name — everything after `..` — with dots turned into + underscores. Keeps a versioned model's version, e.g. `model.shop.dim.v2` -> `dim_v2`. + """ + return "_".join(unique_id.split(".")[2:]) + + +def _source_key(parts: list[str], with_package: bool) -> str: + """ + Test-task key for a source (`source...
`): `_
_test`, + package-prefixed (`__
_test`) when disambiguating a collision. + """ + prefix = f"{parts[1]}_" if with_package else "" + return f"{prefix}{parts[2]}_{parts[3]}_test" + + +def _sanitized_id(unique_id: str) -> str: + """Fallback key for an unhandled resource type: the sanitized `unique_id` (dots -> underscores).""" + return unique_id.replace(".", "_") + + +# Minimum dot-separated segment count required to build a key for each resource type. dbt ids are +# `..[...]`; sources are `source...
`. +_MIN_SEGMENTS = {"model": 3, "seed": 3, "snapshot": 3, "test": 3, "source": 4} + + +def _split_unique_id(unique_id: str) -> list[str]: + """ + Splits a dbt node `unique_id` into its dot-separated parts, validating it has enough segments + for its resource type. Raises `ValueError` with a clear message on a malformed id (e.g. a + name-less `model.pkg`, or a source missing its table). + """ + parts = unique_id.split(".") + minimum = _MIN_SEGMENTS.get(parts[0]) + if minimum is not None and len(parts) < minimum: + raise ValueError(f"Malformed dbt node id {unique_id!r}: expected at least {minimum} dot-separated segments.") + return parts + + +def generate_task_key(unique_id: str) -> str: + """ + Builds a readable Databricks task key from a dbt node `unique_id`. + + The key is based on the dbt resource *name* (not the fully-qualified id), so the package + prefix and dbt's test-name hash never appear, with the resource type as a suffix: + + * `model.shop.orders` -> `orders_model` + * `seed.shop.countries` -> `countries_seed` + * `snapshot.shop.orders_snap` -> `orders_snap_snapshot` + * `test.shop.unique_orders_id.9a1` -> `unique_orders_id_test` (hash dropped) + * `source.shop.raw.customers` -> `raw_customers_test` + + Distinct nodes can map to the same plain key (the same custom test name on two models, a + model name reused across packages), so job generation resolves keys through + `build_task_key_maps`, which keeps these plain keys and disambiguates only actual + collisions. Over-long test keys are truncated and disambiguated with dbt's hash to stay + within the task-key length limit. + """ + parts = _split_unique_id(unique_id) + resource_type = parts[0] + + if resource_type in _SUFFIXED_TYPES: + return f"{_resource_name(unique_id)}_{resource_type}" + + if resource_type == "source": + # A source only ever surfaces as a test task: source...
. + return _source_key(parts, with_package=False) + + if resource_type == "test": + # test..[.] -> _test (drop dbt's uniqueness hash). + test_hash = parts[3] if len(parts) > 3 else "" + return _bounded_test_key(parts[2], test_hash) + + # Unknown type: fall back to the sanitized id (still unique). + return _sanitized_id(unique_id) + + +def bundled_test_key(unique_id: str) -> str: + """ + Key for the single `dbt test` task that gates a tested resource in bundled mode: + `model.shop.orders` -> `orders_test`; `source.shop.raw.customers` -> `raw_customers_test`. + """ + parts = _split_unique_id(unique_id) + if parts[0] == "source": + return _source_key(parts, with_package=False) + return f"{_resource_name(unique_id)}_test" + + +def build_task_key_maps( + task_ids: Iterable[str], bundled_test_ids: Iterable[str] = () +) -> tuple[dict[str, str], dict[str, str]]: + """ + Assigns every task-producing dbt node a unique Databricks task key. + + `task_ids` are nodes that become their own task (keys from `generate_task_key`); + `bundled_test_ids` are tested resources that additionally get a bundled test task in + bundled mode (keys from `bundled_test_key`). Plain keys are kept untouched unless + several nodes claim the same key; each claimant of a contested key then gets a + disambiguated key with dbt's test hash (or the package name, when there is no hash) folded + in, falling back to the sanitized `unique_id`. Every assigned key is passed through + `_reserve`, which enforces both uniqueness and the `MAX_TASK_KEY_LENGTH` limit, so the + returned keys are always unique and within length — a valid dbt project cannot fail + deployment with a duplicate or over-long task key. + + Returns `(task_keys, bundled_test_keys)`, both keyed by `unique_id`. + """ + claims: dict[str, list[tuple[str, bool]]] = {} + for uid in task_ids: + claims.setdefault(generate_task_key(uid), []).append((uid, False)) + for uid in bundled_test_ids: + claims.setdefault(bundled_test_key(uid), []).append((uid, True)) + + task_keys: dict[str, str] = {} + bundled_test_keys: dict[str, str] = {} + taken: set[str] = set() + # Both passes walk their claims in sorted order rather than manifest order. `_reserve`'s + # numeric suffix is handed out first-come, so iterating in manifest order would let the + # *arrangement* of the manifest decide which of two colliding nodes keeps the readable key — + # adding or renaming an unrelated model could then silently swap two task keys, repointing + # per-task run history and task-level alerts at a different model. Sorting makes every + # assigned key a function of the node ids alone. + ordered_claims = sorted((key, sorted(claimants)) for key, claimants in claims.items()) + # Assign uncontested keys first so a contested claimant's fallback never steals a plain key. + for key, claimants in ordered_claims: + if len(claimants) == 1: + uid, is_bundled = claimants[0] + (bundled_test_keys if is_bundled else task_keys)[uid] = _reserve(key, taken) + for key, claimants in ordered_claims: + if len(claimants) == 1: + continue + for uid, is_bundled in claimants: + candidate = _disambiguated_bundled_test_key(uid) if is_bundled else _disambiguated_task_key(uid) + (bundled_test_keys if is_bundled else task_keys)[uid] = _reserve(candidate, taken) + return task_keys, bundled_test_keys + + +def _reserve(candidate: str, taken: set[str]) -> str: + """ + Returns a unique task key at most `MAX_TASK_KEY_LENGTH` characters long, records it in + `taken`, and never returns the same key twice. + + `candidate` is first bounded to the length limit (its tail replaced by a short hash of the + full candidate so distinct over-long keys stay distinct). If the bounded key is already + taken, a numeric suffix is appended and the result re-bounded, until an unused key is found. + """ + key = _bounded(candidate) + if key not in taken: + taken.add(key) + return key + counter = 2 + while True: + key = _bounded(f"{candidate}_{counter}") + if key not in taken: + taken.add(key) + return key + counter += 1 + + +def _bounded(key: str) -> str: + """ + Returns `key` unchanged if within `MAX_TASK_KEY_LENGTH`, otherwise truncates it and appends + a short hash of the full key so distinct over-long keys map to distinct bounded keys. + """ + if len(key) <= MAX_TASK_KEY_LENGTH: + return key + digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:8] + return f"{key[: MAX_TASK_KEY_LENGTH - len(digest) - 1]}_{digest}" + + +def _disambiguated_task_key(unique_id: str) -> str: + """ + Key for a node whose plain key collides with another node's: dbt's test hash (or the + package name, when there is no hash) is folded in to keep the key unique yet readable, + e.g. `test.shop.dup_check.9a1` -> `dup_check_9a1_test`, `model.pkg.orders` -> + `pkg_orders_model`. + """ + parts = unique_id.split(".") + resource_type = parts[0] + if resource_type in _SUFFIXED_TYPES: + return f"{parts[1]}_{_resource_name(unique_id)}_{resource_type}" + if resource_type == "source": + return _source_key(parts, with_package=True) + if resource_type == "test": + if len(parts) > 3: + return _hashed_test_key(parts[2], parts[3]) + return f"{parts[1]}_{parts[2]}_test" + return _sanitized_id(unique_id) + + +def _disambiguated_bundled_test_key(unique_id: str) -> str: + """Package-prefixed variant of `bundled_test_key`, for contested bundled test keys.""" + parts = unique_id.split(".") + if parts[0] == "source": + return _source_key(parts, with_package=True) + return f"{parts[1]}_{_resource_name(unique_id)}_test" + + +def _bounded_test_key(test_name: str, test_hash: str) -> str: + """`_test`, truncated and hash-disambiguated if it exceeds the key length limit.""" + key = f"{test_name}_test" + if len(key) <= MAX_TASK_KEY_LENGTH: + return key + return _hashed_test_key(test_name, test_hash) + + +def _hashed_test_key(test_name: str, test_hash: str) -> str: + """`__test`, truncating the name so the hash survives the length limit.""" + tail = (f"_{test_hash}" if test_hash else "") + "_test" + if len(test_name) + len(tail) <= MAX_TASK_KEY_LENGTH: + return test_name + tail + return test_name[: MAX_TASK_KEY_LENGTH - len(tail)] + tail + + +def read_dbt_manifest(path: str) -> dict: + """ + Reads a dbt manifest JSON file and returns its parsed content. + + Args: + path (str): Path to the manifest file. + + Returns: + dict: Parsed manifest content. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the file is not valid JSON, or is valid JSON that is not an object. A non-object + (list, scalar, null) would parse here and only fail later as an AttributeError on + `manifest.get(...)`, which `main` does not catch — a traceback instead of a clean `error:`. + """ + try: + with open(path, "r", encoding="utf-8") as file: + manifest = json.load(file) + except FileNotFoundError as e: + raise FileNotFoundError(f"Manifest file not found: {path}. Details: {e}") from e + except json.JSONDecodeError as e: + raise ValueError(f"Error parsing JSON from manifest file: {path}. Details: {e}") from e + if not isinstance(manifest, dict): + raise ValueError(f"Manifest file {path} must contain a JSON object, got {type(manifest).__name__}.") + return manifest diff --git a/contrib/dbt_factory/src/macros/.gitkeep b/contrib/dbt_factory/src/macros/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/models/example/orders_daily.sql b/contrib/dbt_factory/src/models/example/orders_daily.sql new file mode 100644 index 00000000..e32736ce --- /dev/null +++ b/contrib/dbt_factory/src/models/example/orders_daily.sql @@ -0,0 +1,17 @@ + +-- This model file defines a materialized view called 'orders_daily' +-- +-- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'materialized_view') }} + +select order_date, count(*) AS number_of_orders + +from {{ ref('orders_raw') }} + +-- During development, only process a smaller range of data +{% if target.name != 'prod' %} +where order_date >= '2019-08-01' and order_date < '2019-09-01' +{% endif %} + +group by order_date diff --git a/contrib/dbt_factory/src/models/example/orders_raw.sql b/contrib/dbt_factory/src/models/example/orders_raw.sql new file mode 100644 index 00000000..8faf8f38 --- /dev/null +++ b/contrib/dbt_factory/src/models/example/orders_raw.sql @@ -0,0 +1,16 @@ +-- This model file defines a streaming table called 'orders_raw' +-- +-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ +-- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'streaming_table') }} + +select + customer_name, + date(timestamp(from_unixtime(try_cast(order_datetime as bigint)))) as order_date, + order_number +from stream read_files( + "/databricks-datasets/retail-org/sales_orders/", + format => "json", + header => true +) diff --git a/contrib/dbt_factory/src/models/example/schema.yml b/contrib/dbt_factory/src/models/example/schema.yml new file mode 100644 index 00000000..c17c810e --- /dev/null +++ b/contrib/dbt_factory/src/models/example/schema.yml @@ -0,0 +1,22 @@ + +version: 2 + +models: + - name: orders_raw + description: "Raw ingested orders" + columns: + - name: customer_name + description: "The name of a customer" + # customer_name repeats across orders, so only a not_null test applies here. + data_tests: + - not_null + + - name: orders_daily + description: "Number of orders by day" + columns: + - name: order_date + description: "The date on which orders took place" + # orders_daily groups by order_date, so it is unique and not null. + data_tests: + - unique + - not_null diff --git a/contrib/dbt_factory/src/seeds/.gitkeep b/contrib/dbt_factory/src/seeds/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/snapshots/.gitkeep b/contrib/dbt_factory/src/snapshots/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/tests/.gitkeep b/contrib/dbt_factory/src/tests/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/target/manifest.json b/contrib/dbt_factory/target/manifest.json new file mode 100644 index 00000000..21822ec5 --- /dev/null +++ b/contrib/dbt_factory/target/manifest.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": "1.11.11", "generated_at": "2026-07-02T17:03:13.876639Z", "invocation_id": "04710f75-f17a-49ce-b19e-779d81afacd1", "invocation_started_at": "2026-07-02T17:03:12.678402Z", "env": {}, "project_name": "dbt_factory", "project_id": "7085a6babca105474633a8d7cb1f5b8d", "user_id": "0aecafb9-78fd-4aa4-aba3-0e8995898cd4", "send_anonymous_usage_stats": true, "adapter_type": "databricks", "quoting": {"database": true, "schema": true, "identifier": true, "column": null}, "run_started_at": "2026-07-02T17:03:12.678594+00:00"}, "nodes": {"model.dbt_factory.orders_raw": {"database": "catalog", "schema": "default", "name": "orders_raw", "resource_type": "model", "package_name": "dbt_factory", "path": "example/orders_raw.sql", "original_file_path": "src/models/example/orders_raw.sql", "unique_id": "model.dbt_factory.orders_raw", "fqn": ["dbt_factory", "example", "orders_raw"], "alias": "orders_raw", "checksum": {"name": "sha256", "checksum": "057e5b384cf1b4d8a5e29453f900cbd74609cbfb070aee02baa0c691f5bd2179"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "streaming_table", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null}, "tags": [], "description": "Raw ingested orders", "columns": {"customer_name": {"name": "customer_name", "description": "The name of a customer", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": {"meta": {}, "tags": []}, "tags": [], "granularity": null, "doc_blocks": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_factory://src/models/example/schema.yml", "build_path": null, "unrendered_config": {"materialized": "streaming_table"}, "created_at": 1783011794.5405571, "relation_name": "`catalog`.`default`.`orders_raw`", "raw_code": "-- This model file defines a streaming table called 'orders_raw'\n--\n-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/\n-- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables\n-- Current limitation: a \"full refresh\" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561.\n{{ config(materialized = 'streaming_table') }}\n\nselect\n customer_name,\n date(timestamp(from_unixtime(try_cast(order_datetime as bigint)))) as order_date,\n order_number\nfrom stream read_files(\n \"/databricks-datasets/retail-org/sales_orders/\",\n format => \"json\",\n header => true\n)", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null}, "model.dbt_factory.orders_daily": {"database": "catalog", "schema": "default", "name": "orders_daily", "resource_type": "model", "package_name": "dbt_factory", "path": "example/orders_daily.sql", "original_file_path": "src/models/example/orders_daily.sql", "unique_id": "model.dbt_factory.orders_daily", "fqn": ["dbt_factory", "example", "orders_daily"], "alias": "orders_daily", "checksum": {"name": "sha256", "checksum": "5ec9641ef90bdd9f6011aea792cb9042e1e21b7ece23f55ca9b2eddb5bff5630"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "materialized_view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null}, "tags": [], "description": "Number of orders by day", "columns": {"order_date": {"name": "order_date", "description": "The date on which orders took place", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": {"meta": {}, "tags": []}, "tags": [], "granularity": null, "doc_blocks": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_factory://src/models/example/schema.yml", "build_path": null, "unrendered_config": {"materialized": "materialized_view"}, "created_at": 1783011794.540988, "relation_name": "`catalog`.`default`.`orders_daily`", "raw_code": "-- This model file defines a materialized view called 'orders_daily'\n--\n-- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables\n-- Current limitation: a \"full refresh\" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561.\n{{ config(materialized = 'materialized_view') }}\n\nselect order_date, count(*) AS number_of_orders\n\nfrom {{ ref('orders_raw') }}\n\n-- During development, only process a smaller range of data\n{% if target.name != 'prod' %}\nwhere order_date >= '2019-08-01' and order_date < '2019-09-01'\n{% endif %}\n\ngroup by order_date", "doc_blocks": [], "language": "sql", "refs": [{"name": "orders_raw", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": ["model.dbt_factory.orders_raw"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": ["order_date"], "time_spine": null}, "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91": {"database": "catalog", "schema": "default_dbt_test__audit", "name": "not_null_orders_raw_customer_name", "resource_type": "test", "package_name": "dbt_factory", "path": "not_null_orders_raw_customer_name.sql", "original_file_path": "src/models/example/schema.yml", "unique_id": "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91", "fqn": ["dbt_factory", "models", "example", "not_null_orders_raw_customer_name"], "alias": "not_null_orders_raw_customer_name", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1783011794.567219, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [{"name": "orders_raw", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_factory.orders_raw"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "customer_name", "file_key_name": "models.orders_raw", "attached_node": "model.dbt_factory.orders_raw", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "customer_name", "model": "{{ get_where_subquery(ref('orders_raw')) }}"}, "namespace": null}}, "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3": {"database": "catalog", "schema": "default_dbt_test__audit", "name": "unique_orders_daily_order_date", "resource_type": "test", "package_name": "dbt_factory", "path": "unique_orders_daily_order_date.sql", "original_file_path": "src/models/example/schema.yml", "unique_id": "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3", "fqn": ["dbt_factory", "models", "example", "unique_orders_daily_order_date"], "alias": "unique_orders_daily_order_date", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1783011794.568104, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [{"name": "orders_daily", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_factory.orders_daily"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "order_date", "file_key_name": "models.orders_daily", "attached_node": "model.dbt_factory.orders_daily", "test_metadata": {"name": "unique", "kwargs": {"column_name": "order_date", "model": "{{ get_where_subquery(ref('orders_daily')) }}"}, "namespace": null}}, "test.dbt_factory.not_null_orders_daily_order_date.e8076bc258": {"database": "catalog", "schema": "default_dbt_test__audit", "name": "not_null_orders_daily_order_date", "resource_type": "test", "package_name": "dbt_factory", "path": "not_null_orders_daily_order_date.sql", "original_file_path": "src/models/example/schema.yml", "unique_id": "test.dbt_factory.not_null_orders_daily_order_date.e8076bc258", "fqn": ["dbt_factory", "models", "example", "not_null_orders_daily_order_date"], "alias": "not_null_orders_daily_order_date", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1783011794.5687559, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [{"name": "orders_daily", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_factory.orders_daily"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "order_date", "file_key_name": "models.orders_daily", "attached_node": "model.dbt_factory.orders_daily", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "order_date", "model": "{{ get_where_subquery(ref('orders_daily')) }}"}, "namespace": null}}}, "sources": {}, "macros": {"macro.dbt_databricks.materialization_materialized_view_databricks": {"name": "materialization_materialized_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialization_materialized_view_databricks", "macro_sql": "{% materialization materialized_view, adapter = 'databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.materialized_view_get_build_sql", "macro.dbt_databricks.execute_no_op", "macro.dbt_databricks.materialized_view_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.962945, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation) %}\n {% set sql = adapter.clean_sql(sql) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_configuration_changes(existing_relation) %}\n\n {# Skip manual REFRESH on no-op re-runs for auto-refreshed modes. #}\n {% if configuration_changes is none %}\n {%- set refresh = adapter.get_config_from_model(config.model).config[\"refresh\"] -%}\n {%- if refresh.auto_refreshed -%}\n {% set build_sql = '' %}\n {%- else -%}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n {%- endif -%}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt_databricks.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9637198, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n\n {{ execute_multiple_statements(build_sql) }}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt_databricks.execute_multiple_statements", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.96409, "supported_languages": null}, "macro.dbt_databricks.materialization_view_databricks": {"name": "materialization_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.materialization_view_databricks", "macro_sql": "{% materialization view, adapter='databricks' -%}\n {{ log(\"MATERIALIZING VIEW\") }}\n {%- set existing_relation = load_relation_with_metadata(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n {% set sql = adapter.clean_sql(sql) %}\n\n {% if adapter.get_behavior_flag_no_warn('use_materialization_v2') %}\n {{ run_pre_hooks() }}\n {% if existing_relation %}\n {% if relation_should_be_altered(existing_relation) %}\n {% set configuration_changes = get_configuration_changes(existing_relation) %}\n {% if configuration_changes and configuration_changes.changes %}\n {% if configuration_changes.requires_full_refresh %}\n {{ log('Using replace_with_view') }}\n {{ replace_with_view(existing_relation, target_relation) }}\n {% else %}\n {{ log('Using alter_view') }}\n {{ log(configuration_changes.changes) }}\n {{ alter_view(target_relation, configuration_changes.changes) }}\n {% endif %}\n {% else %}\n {{ execute_no_op(target_relation) }}\n {% endif %}\n {% else %}\n {{ replace_with_view(existing_relation, target_relation) }}\n {% endif %}\n {% else %}\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n {{ apply_tags(target_relation, tags) }}\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n {% endif %}\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {{ run_post_hooks() }}\n\n {% else %}\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if existing_relation is not none and not existing_relation.is_view -%}\n {{ handle_existing_table(should_full_refresh(), existing_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {{ run_hooks(post_hooks) }}\n {% endif %}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.load_relation_with_metadata", "macro.dbt_databricks.run_pre_hooks", "macro.dbt_databricks.relation_should_be_altered", "macro.dbt_databricks.get_configuration_changes", "macro.dbt_databricks.replace_with_view", "macro.dbt_databricks.alter_view", "macro.dbt_databricks.execute_no_op", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.run_post_hooks", "macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.965732, "supported_languages": ["sql"]}, "macro.dbt_databricks.replace_with_view": {"name": "replace_with_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.replace_with_view", "macro_sql": "{% macro replace_with_view(existing_relation, target_relation) %}\n {% set sql = adapter.clean_sql(sql) %}\n {% set tags = config.get('databricks_tags') %}\n {{ execute_multiple_statements(get_replace_sql(existing_relation, target_relation, sql)) }}\n {%- do apply_tags(target_relation, tags) -%}\n \n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.execute_multiple_statements", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.966026, "supported_languages": null}, "macro.dbt_databricks.relation_should_be_altered": {"name": "relation_should_be_altered", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.relation_should_be_altered", "macro_sql": "{% macro relation_should_be_altered(existing_relation) %}\n {% set update_via_alter = config.get('view_update_via_alter', False) | as_bool %}\n {% if (existing_relation.is_view or existing_relation.is_metric_view) and update_via_alter %}\n {% if existing_relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Cannot update a view in the Hive metastore via ALTER VIEW. Please set `view_update_via_alter: false` in your model configuration.\") }}\n {% endif %}\n {{ return(True) }}\n {% endif %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.966244, "supported_languages": null}, "macro.dbt_databricks.materialization_table_databricks": {"name": "materialization_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_databricks.materialization_table_databricks", "macro_sql": "{% materialization table, adapter = 'databricks', supported_languages=['sql', 'python'] %}\n {{ log(\"MATERIALIZING TABLE\") }}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n {%- set safe_create = config.get('use_safer_relation_operations', False) %}\n {% set existing_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) %}\n {% set target_relation = this.incorporate(type='table') %}\n {% set compiled_code = adapter.clean_sql(compiled_code) %}\n\n {% if adapter.get_behavior_flag_no_warn('use_materialization_v2') %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set staging_relation = make_staging_relation(target_relation) %}\n\n {{ run_pre_hooks() }}\n\n {% call statement('main', language=language) %}\n {{ get_create_intermediate_table(intermediate_relation, compiled_code, language) }}\n {% endcall %}\n {% if not existing_relation %}\n {{ create_table_at(target_relation, intermediate_relation, compiled_code) }}\n {% else %}\n {% if safe_create and existing_relation.can_be_renamed %}\n {{ safe_relation_replace(existing_relation, staging_relation, intermediate_relation, compiled_code) }}\n {% else %}\n {% if existing_relation and (existing_relation.type != 'table' or not (existing_relation.can_be_replaced and adapter.resolve_file_format(config) in ('delta', 'iceberg'))) -%}\n {{ adapter.drop_relation(existing_relation) }}\n {%- endif %}\n {{ create_table_at(target_relation, intermediate_relation, compiled_code) }}\n {% endif %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {{ apply_grants(target_relation, grant_config, should_revoke) }}\n {% do optimize(target_relation) %}\n\n {% if language == 'python' %}\n {{ drop_relation_if_exists(intermediate_relation) }}\n {% endif %}\n \n {{ run_post_hooks() }}\n {% else %}\n {{ run_hooks(pre_hooks) }}\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if existing_relation and (existing_relation.type != 'table' or not (existing_relation.can_be_replaced and adapter.resolve_file_format(config) in ('delta', 'iceberg'))) -%}\n {{ adapter.drop_relation(existing_relation) }}\n {%- endif %}\n\n -- build model\n\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% if language==\"python\" %}\n {% do apply_tblproperties(target_relation, tblproperties) %}\n {% endif %}\n {%- do apply_tags(target_relation, tags) -%}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {% endif %}\n {{ return({'relations': [target_relation]})}}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt_databricks.make_staging_relation", "macro.dbt_databricks.run_pre_hooks", "macro.dbt.statement", "macro.dbt_databricks.get_create_intermediate_table", "macro.dbt_databricks.create_table_at", "macro.dbt_databricks.safe_relation_replace", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.optimize", "macro.dbt.drop_relation_if_exists", "macro.dbt_databricks.run_post_hooks", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags", "macro.dbt.persist_docs", "macro.dbt_databricks.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9679391, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.run_pre_hooks": {"name": "run_pre_hooks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt_databricks.run_pre_hooks", "macro_sql": "{% macro run_pre_hooks() %}\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.968092, "supported_languages": null}, "macro.dbt_databricks.run_post_hooks": {"name": "run_post_hooks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt_databricks.run_post_hooks", "macro_sql": "{% macro run_post_hooks() %}\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.968184, "supported_languages": null}, "macro.dbt_databricks.materialization_snapshot_databricks": {"name": "materialization_snapshot_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.materialization_snapshot_databricks", "macro_sql": "{% materialization snapshot, adapter='databricks' %}\n {%- set target_table = model.get('alias', model.get('name')) -%}\n {% set sql = adapter.clean_sql(sql) %}\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tags = config.get('databricks_tags') -%}\n\n {% set target_relation_exists, target_relation = databricks__get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table',\n needs_information=True) -%}\n\n {%- if file_format not in ['delta', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, adapter.clean_sql(model['compiled_code'])) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}\n {% if unique_key | is_list %}\n {% for key in strategy.unique_key %}\n {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }}\n {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }}\n {% endfor %}\n {% endif %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(column.quoted) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n \n {%- if tags -%}\n {%- do apply_tags(target_relation, tags) -%}\n {%- endif -%}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.970536, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_streaming_table_databricks": {"name": "materialization_streaming_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.materialization_streaming_table_databricks", "macro_sql": "{% materialization streaming_table, adapter='databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.StreamingTable) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% set build_sql = streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ execute_no_op(target_relation) }}\n {% else %}\n {{ streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.streaming_table_get_build_sql", "macro.dbt_databricks.execute_no_op", "macro.dbt_databricks.streaming_table_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.971122, "supported_languages": ["sql"]}, "macro.dbt_databricks.streaming_table_get_build_sql": {"name": "streaming_table_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_get_build_sql", "macro_sql": "{% macro streaming_table_get_build_sql(existing_relation, target_relation) %}\n {% set sql = adapter.clean_sql(sql) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_streaming_table_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_streaming_table %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_configuration_changes(existing_relation) %}\n {# Skip manual REFRESH on no-op re-runs for auto-refreshed modes. #}\n {% if configuration_changes is none %}\n {%- set refresh = adapter.get_config_from_model(config.model).config[\"refresh\"] -%}\n {%- if refresh.auto_refreshed -%}\n {% set build_sql = '' %}\n {%- else -%}\n {{ log(\"REFRESHING STREAMING TABLE: \" ~ target_relation) }}\n {% set build_sql = refresh_streaming_table(target_relation, sql) %}\n {%- endif -%}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_streaming_table_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_configuration_changes", "macro.dbt_databricks.refresh_streaming_table", "macro.dbt_databricks.get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.971827, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_build_sql": {"name": "streaming_table_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_build_sql", "macro_sql": "{% macro streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n\n {{ execute_multiple_statements(build_sql) }}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt_databricks.execute_multiple_statements", "macro.dbt_databricks.apply_tags", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_column_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.972262, "supported_languages": null}, "macro.dbt_databricks.materialization_metric_view_databricks": {"name": "materialization_metric_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/metric_view.sql", "original_file_path": "macros/materializations/metric_view.sql", "unique_id": "macro.dbt_databricks.materialization_metric_view_databricks", "macro_sql": "{% materialization metric_view, adapter='databricks' -%}\n {%- set existing_relation = load_relation_with_metadata(this) -%}\n {%- set target_relation = this.incorporate(type='metric_view') -%}\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n {% set sql = adapter.clean_sql(sql) %}\n\n {{ run_pre_hooks() }}\n\n {% if existing_relation %}\n {#- Only use alter path if existing relation is actually a metric_view -#}\n {% if existing_relation.is_metric_view and relation_should_be_altered(existing_relation) %}\n {% set configuration_changes = get_configuration_changes(existing_relation) %}\n {% if configuration_changes and configuration_changes.changes %}\n {% if configuration_changes.requires_full_refresh %}\n {{ replace_with_metric_view(existing_relation, target_relation) }}\n {% else %}\n {{ alter_metric_view(target_relation, configuration_changes.changes) }}\n {% endif %}\n {% else %}\n {# No changes detected - run a no-op statement for dbt tracking #}\n {% call statement('main') %}\n select 1\n {% endcall %}\n {% endif %}\n {% else %}\n {{ replace_with_metric_view(existing_relation, target_relation) }}\n {% endif %}\n {% else %}\n {% call statement('main') -%}\n {{ get_create_metric_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n {{ apply_tags(target_relation, tags) }}\n {% endif %}\n\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke(existing_relation, full_refresh_mode=True)) %}\n\n {{ run_post_hooks() }}\n\n {{ return({'relations': [target_relation]}) }}\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.load_relation_with_metadata", "macro.dbt_databricks.run_pre_hooks", "macro.dbt_databricks.relation_should_be_altered", "macro.dbt_databricks.get_configuration_changes", "macro.dbt_databricks.replace_with_metric_view", "macro.dbt_databricks.alter_metric_view", "macro.dbt.statement", "macro.dbt_databricks.get_create_metric_view_as_sql", "macro.dbt_databricks.apply_tags", "macro.dbt.apply_grants", "macro.dbt.should_revoke", "macro.dbt_databricks.run_post_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.973301, "supported_languages": ["sql"]}, "macro.dbt_databricks.databricks__snapshot_merge_sql": {"name": "databricks__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot_helpers.sql", "original_file_path": "macros/materializations/snapshot_helpers.sql", "unique_id": "macro.dbt_databricks.databricks__snapshot_merge_sql", "macro_sql": "{% macro databricks__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n {# Get the hard_deletes configuration from config #}\n {%- set hard_deletes = config.get('hard_deletes', 'ignore') -%}\n {%- set invalidate_hard_deletes = (hard_deletes == 'invalidate') -%}\n\n {# Get column names configuration #}\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {%- if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {%- else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {%- endif %}\n on DBT_INTERNAL_SOURCE.{{ adapter.quote(columns.dbt_scd_id) }} = DBT_INTERNAL_DEST.{{ adapter.quote(columns.dbt_scd_id) }}\n when matched\n {%- if config.get(\"dbt_valid_to_current\") %}\n and ( DBT_INTERNAL_DEST.{{ adapter.quote(columns.dbt_valid_to) }} = {{ config.get('dbt_valid_to_current') }} or\n DBT_INTERNAL_DEST.{{ adapter.quote(columns.dbt_valid_to) }} is null )\n {%- else %}\n and DBT_INTERNAL_DEST.{{ adapter.quote(columns.dbt_valid_to) }} is null\n {%- endif %}\n and DBT_INTERNAL_SOURCE.{{ adapter.quote('dbt_change_type') }} in ('update', 'delete')\n then update\n set {{ adapter.quote(columns.dbt_valid_to) }} = DBT_INTERNAL_SOURCE.{{ adapter.quote(columns.dbt_valid_to) }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.{{ adapter.quote('dbt_change_type') }} = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n ;\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.974571, "supported_languages": null}, "macro.dbt_databricks.databricks__create_columns": {"name": "databricks__create_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot_helpers.sql", "original_file_path": "macros/materializations/snapshot_helpers.sql", "unique_id": "macro.dbt_databricks.databricks__create_columns", "macro_sql": "{% macro databricks__create_columns(relation, columns) %}\n {%- if columns|length > 0 %}\n {%- call statement() %}\n alter table {{ relation }} add columns (\n {%- for column in columns %}\n {{ adapter.quote(column.name) }} {{ column.data_type }} {{- ',' if not loop.last -}}\n {%- endfor %}\n );\n {%- endcall %}\n {%- endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.974782, "supported_languages": null}, "macro.dbt_databricks.databricks__build_snapshot_table": {"name": "databricks__build_snapshot_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot_helpers.sql", "original_file_path": "macros/materializations/snapshot_helpers.sql", "unique_id": "macro.dbt_databricks.databricks__build_snapshot_table", "macro_sql": "{% macro databricks__build_snapshot_table(strategy, sql) %}\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n {%- set hard_deletes = strategy.hard_deletes -%}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }}\n {%- if hard_deletes == 'new_record' -%}\n , false as {{ columns.dbt_is_deleted }}\n {%- endif %}\n from (\n {{ sql }}\n ) sbq\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.get_dbt_valid_to_current"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.975029, "supported_languages": null}, "macro.dbt_databricks.databricks__snapshot_staging_table": {"name": "databricks__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot_helpers.sql", "original_file_path": "macros/materializations/snapshot_helpers.sql", "unique_id": "macro.dbt_databricks.databricks__snapshot_staging_table", "macro_sql": "{% macro databricks__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n {% if strategy.hard_deletes == 'new_record' %}\n {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %}\n {% endif %}\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from {{ target_relation }}\n where\n {% if config.get('dbt_valid_to_current') %}\n {% set source_unique_key = columns.dbt_valid_to | trim %}\n {% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\n {# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #}\n ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )\n {% else %}\n {{ columns.dbt_valid_to }} is null\n {% endif %}\n\n ),\n\n insertions_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n\n deletes_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n {%- if strategy.hard_deletes == 'new_record' -%}\n , false as {{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from insertions_source_data as source_data\n left outer join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"snapshotted_data\") }}\n or ({{ unique_key_is_not_null(strategy.unique_key, \"snapshotted_data\") }} and (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = true {% endif %}\n )\n\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from updates_source_data as source_data\n join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = true {% endif %}\n )\n )\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n ,\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n and not (\n -- avoid updating the record's valid_to if the latest entry is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = true\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n {%- endif %}\n )\n {%- endif %}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n {# Databricks-specific: Extract column names from agate.Row tuples #}\n {% set target_columns_raw = get_columns_in_relation(target_relation) %}\n {% set snapshotted_cols = [] %}\n {% for row in target_columns_raw %}\n {# agate.Row is a tuple: (col_name, data_type, comment) #}\n {# Filter out Databricks metadata rows (starting with # or empty) #}\n {% set col_name = row[0] %}\n {% if col_name and not col_name.startswith('#') %}\n {% do snapshotted_cols.append(col_name) %}\n {% endif %}\n {% endfor %}\n {% set source_sql_cols = get_column_schema_from_query(source_sql) %}\n ,\n deletion_records as (\n\n select\n 'insert' as dbt_change_type,\n {#\n If a column has been added to the source it won't yet exist in the\n snapshotted table so we insert a null value as a placeholder for the column.\n #}\n {%- for col in source_sql_cols -%}\n {%- if col.name in snapshotted_cols -%}\n snapshotted_data.{{ adapter.quote(col.column) }},\n {%- else -%}\n NULL as {{ adapter.quote(col.column) }},\n {%- endif -%}\n {% endfor -%}\n {%- if strategy.unique_key | is_list -%}\n {%- for key in strategy.unique_key -%}\n snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},\n {% endfor -%}\n {%- else -%}\n snapshotted_data.dbt_unique_key as dbt_unique_key,\n {% endif -%}\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},\n {{ new_scd_id }} as {{ columns.dbt_scd_id }},\n true as {{ columns.dbt_is_deleted }}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n and not (\n -- avoid inserting a new record if the latest one is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = true\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletes\n {%- endif %}\n {%- if strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletion_records\n {%- endif %}\n\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments", "macro.dbt.snapshot_get_time", "macro.dbt.unique_key_fields", "macro.dbt.equals", "macro.dbt.get_dbt_valid_to_current", "macro.dbt.unique_key_join_on", "macro.dbt.unique_key_is_null", "macro.dbt.unique_key_is_not_null", "macro.dbt.get_columns_in_relation", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9774451, "supported_languages": null}, "macro.dbt_databricks.dbt_databricks_validate_get_file_format": {"name": "dbt_databricks_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro_sql": "{% macro dbt_databricks_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9778838, "supported_languages": null}, "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy": {"name": "dbt_databricks_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_delta_only_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via warehouse\n Use the 'merge' or 'replace_where' strategy instead\n {%- endset %}\n\n {% set missing_unique_key_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n This strategy requires 'unique_key' to be configured\n {%- endset %}\n\n {% if raw_strategy not in adapter.valid_incremental_strategies() %}\n {{ log(\"WARNING - You are using an unsupported incremental strategy: \" ~ raw_strategy) }}\n {{ log(\"You can ignore this warning if you are using a custom incremental strategy\") }}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy in ('replace_where', 'microbatch', 'delete+insert') and file_format not in ['delta'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy == 'delete+insert' and config.get('unique_key') is none %}\n {% do exceptions.raise_compiler_error(missing_unique_key_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.97838, "supported_languages": null}, "macro.dbt_databricks.get_incremental_strategy": {"name": "get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_incremental_strategy", "macro_sql": "{% macro get_incremental_strategy(file_format) %}\n {% set raw_strategy = config.get('incremental_strategy') or 'merge' %}\n {% do return(dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.979957, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_default_sql": {"name": "databricks__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_default_sql", "macro_sql": "{% macro databricks__get_incremental_default_sql(arg_dict) %}\n {{ return(get_incremental_merge_sql(arg_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980026, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_append_sql": {"name": "databricks__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_append_sql", "macro_sql": "{% macro databricks__get_incremental_append_sql(arg_dict) %}\n {% do return(get_insert_into_sql(arg_dict[\"temp_relation\"], arg_dict[\"target_relation\"])) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980118, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_replace_where_sql": {"name": "databricks__get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_replace_where_sql", "macro_sql": "{% macro databricks__get_incremental_replace_where_sql(arg_dict) %}\n {% do return(get_replace_where_sql(arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980194, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_delete_insert_sql": {"name": "databricks__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_delete_insert_sql", "macro_sql": "{% macro databricks__get_incremental_delete_insert_sql(arg_dict) %}\n {% do return(get_delete_insert_sql(arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980268, "supported_languages": null}, "macro.dbt_databricks.get_incremental_replace_where_sql": {"name": "get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_incremental_replace_where_sql", "macro_sql": "{% macro get_incremental_replace_where_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_replace_where_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9803588, "supported_languages": null}, "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql": {"name": "databricks__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql", "macro_sql": "{% macro databricks__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}\n {{ return(get_insert_overwrite_sql(source, target)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980452, "supported_languages": null}, "macro.dbt_databricks.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation) %}\n {%- if adapter.is_cluster() -%}\n {#-- Clusters: Check DBR capability for REPLACE ON (DBR 17.1+) --#}\n {%- if adapter.has_dbr_capability('replace_on') -%}\n {{ get_insert_replace_on_sql(source_relation, target_relation) }}\n {%- else -%}\n {%- if adapter.behavior.use_replace_on_for_insert_overwrite -%}\n {{ exceptions.warn(\"insert_overwrite: use_replace_on_for_insert_overwrite is enabled but this cluster's DBR version does not support REPLACE ON (requires DBR 17.1+). Falling back to legacy INSERT OVERWRITE.\") }}\n {%- endif -%}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {%- if has_insert_by_name %} by name{% endif %}\n select * from {{ source_relation }}\n {%- endif -%}\n {%- else -%}\n {%- if adapter.behavior.use_replace_on_for_insert_overwrite -%}\n {{ get_insert_replace_on_sql(source_relation, target_relation) }}\n {%- else -%}\n {#-- Behavior flag disabled, use legacy DPO INSERT OVERWRITE --#}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {%- if has_insert_by_name %} by name{% endif %}\n select * from {{ source_relation }}\n {%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_replace_on_sql", "macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980876, "supported_languages": null}, "macro.dbt_databricks.add_columns_to_list": {"name": "add_columns_to_list", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.add_columns_to_list", "macro_sql": "{% macro add_columns_to_list(target_list, columns) %}\n {#-- Local helper to add columns to the target list, converting string to list if needed --#}\n {%- if columns -%}\n {%- if columns is string -%}\n {%- do target_list.append(columns) -%}\n {%- else -%}\n {%- for col in columns -%}\n {%- do target_list.append(col) -%}\n {%- endfor -%}\n {%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.981042, "supported_languages": null}, "macro.dbt_databricks.get_insert_replace_on_sql": {"name": "get_insert_replace_on_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_replace_on_sql", "macro_sql": "{% macro get_insert_replace_on_sql(source_relation, target_relation) %}\n {%- set partition_by = config.get('partition_by') -%}\n {%- set liquid_clustered_by = config.get('liquid_clustered_by') -%}\n {%- set replace_columns = [] -%}\n \n {#-- If both partition_by and liquid_clustered_by are defined, it will fail before this point with a SPECIFY_CLUSTER_BY_WITH_PARTITIONED_BY_IS_NOT_ALLOWED error from Databricks --#}\n {%- do add_columns_to_list(replace_columns, partition_by) -%}\n {%- do add_columns_to_list(replace_columns, liquid_clustered_by) -%}\n \n {%- if replace_columns -%}\n {%- set replace_conditions = [] -%}\n {%- for col in replace_columns -%}\n {%- do replace_conditions.append('t.' ~ col ~ ' <=> s.' ~ col) -%}\n {%- endfor -%}\n {%- set replace_conditions_csv = replace_conditions | join(' AND ') -%}\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) | map(attribute=\"name\") | list -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) | map(attribute=\"name\") | list -%}\n {%- set source_cols_lower = source_columns | map('lower') | list -%}\n {%- set select_columns = [] -%}\n {%- for dest_col in dest_columns -%}\n {%- set dest_col_lower = dest_col | lower -%}\n {%- set matched_col = namespace(value=none) -%}\n {%- for src_col in source_columns -%}\n {%- if src_col | lower == dest_col_lower and matched_col.value is none -%}\n {%- set matched_col.value = src_col -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if matched_col.value is not none -%}\n {%- do select_columns.append(matched_col.value) -%}\n {%- else -%}\n {%- do select_columns.append('NULL as ' ~ dest_col) -%}\n {%- endif -%}\n {%- endfor -%}\n insert into table {{ target_relation }} AS t\n replace on ({{ replace_conditions_csv }})\n (select {{ select_columns | join(', ') }} from {{ source_relation }}) AS s\n {%- else -%}\n {#-- Fallback to regular insert overwrite if no partitioning nor liquid clustering defined --#}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n insert overwrite table {{ target_relation }}\n {%- if has_insert_by_name %} by name{% endif %}\n select * from {{ source_relation }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.add_columns_to_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.981857, "supported_languages": null}, "macro.dbt_databricks.get_replace_where_sql": {"name": "get_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_replace_where_sql", "macro_sql": "{% macro get_replace_where_sql(args_dict) -%}\n {%- set predicates = args_dict['incremental_predicates'] -%}\n {%- set target_relation = args_dict['target_relation'] -%}\n {%- set temp_relation = args_dict['temp_relation'] -%}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\nINSERT INTO {{ target_relation.render() }}\n{%- if has_insert_by_name %} BY NAME{% endif %}\n{%- if predicates %}\n {%- if predicates is sequence and predicates is not string %}\n REPLACE WHERE {{ predicates | join(' and ') }}\n {%- else %}\n REPLACE WHERE {{ predicates }}\n {%- endif %}\n{%- endif %}\n TABLE {{ temp_relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9822028, "supported_languages": null}, "macro.dbt_databricks.get_delete_insert_sql": {"name": "get_delete_insert_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_delete_insert_sql", "macro_sql": "{% macro get_delete_insert_sql(arg_dict) -%}\n {%- set source_relation = arg_dict.get('temp_relation') -%}\n {%- set target_relation = arg_dict.get('target_relation') -%}\n {%- set incremental_predicates = config.get('incremental_predicates') -%}\n {%- set target_columns = (adapter.get_columns_in_relation(target_relation) | map(attribute='quoted') | list) -%}\n {%- set unique_key = config.require('unique_key') -%}\n {% do return(delete_insert_sql_impl(source_relation, target_relation, target_columns, unique_key, incremental_predicates)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.delete_insert_sql_impl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.98253, "supported_languages": null}, "macro.dbt_databricks.delete_insert_sql_impl": {"name": "delete_insert_sql_impl", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.delete_insert_sql_impl", "macro_sql": "{% macro delete_insert_sql_impl(source_relation, target_relation, target_columns, unique_key, incremental_predicates) %}\n {# we use target_columns to select from source to avoid potential column order issues during insert #}\n {%- set target_cols_csv = target_columns | join(', ') -%}\n {%- set predicates -%}\n {%- if incremental_predicates is sequence and incremental_predicates is not string -%}\n where {{ incremental_predicates | join(' and ') }}\n {%- elif incremental_predicates is string and incremental_predicates is not none -%}\n where {{ incremental_predicates }}\n {%- endif -%}\n {%- endset -%}\n {%- set unique_keys = unique_key if unique_key is sequence and unique_key is not string else [unique_key] -%}\n \n {%- if adapter.has_dbr_capability('replace_on') -%}\n {#-- DBR 17.1+: Use efficient REPLACE ON syntax --#}\n {%- set replace_on_expr = [] -%}\n {%- for key in unique_keys -%}\n {%- do replace_on_expr.append('target.' ~ key ~ ' <=> temp.' ~ key) -%}\n {%- endfor -%}\n {%- set replace_on_expr = replace_on_expr | join(' and ') -%}\n insert into table {{ target_relation }} as target\nreplace on ({{ replace_on_expr }})\n(select {{ target_cols_csv }}\n from {{ source_relation }} {{ predicates }}) as temp\n {%- else -%}\n {#-- DBR < 17.1: Fallback to DELETE FROM + INSERT INTO --#}\n {% do return(delete_insert_legacy_sql(source_relation, target_relation, target_columns, unique_keys, incremental_predicates)) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.delete_insert_legacy_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.983084, "supported_languages": null}, "macro.dbt_databricks.delete_insert_legacy_sql": {"name": "delete_insert_legacy_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.delete_insert_legacy_sql", "macro_sql": "{% macro delete_insert_legacy_sql(source_relation, target_relation, target_columns, unique_keys, incremental_predicates) %}\n {#-- Legacy implementation for DBR < 17.1 using DELETE FROM + INSERT INTO --#}\n {#-- Returns a list of SQL statements to be executed via execute_multiple_statements --#}\n {%- set target_cols_csv = target_columns | join(', ') -%}\n {%- set statements = [] -%}\n \n {#-- Build WHERE clause for DELETE statement --#}\n {%- set delete_conditions = [] -%}\n {%- for key in unique_keys -%}\n {%- do delete_conditions.append(target_relation ~ '.' ~ key ~ ' IN (SELECT ' ~ key ~ ' FROM ' ~ source_relation ~ ')') -%}\n {%- endfor -%}\n \n {#-- Add incremental predicates to DELETE if specified --#}\n {%- if incremental_predicates is sequence and incremental_predicates is not string -%}\n {%- for predicate in incremental_predicates -%}\n {%- do delete_conditions.append(predicate) -%}\n {%- endfor -%}\n {%- elif incremental_predicates is string and incremental_predicates is not none -%}\n {%- do delete_conditions.append(incremental_predicates) -%}\n {%- endif -%}\n \n {#-- Step 1: DELETE matching rows --#}\n {%- set delete_sql -%}\ndelete from {{ target_relation }}\nwhere {{ delete_conditions | join('\\n and ') }}\n {%- endset -%}\n {%- do statements.append(delete_sql) -%}\n\n {#-- Step 2: INSERT new rows --#}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n {%- set insert_sql -%}\ninsert into {{ target_relation }}{% if has_insert_by_name %} by name{% endif %}\n\nselect {{ target_cols_csv }}\nfrom {{ source_relation }}\n {%- if incremental_predicates is sequence and incremental_predicates is not string %}\nwhere {{ incremental_predicates | join(' and ') }}\n {%- elif incremental_predicates is string and incremental_predicates is not none %}\nwhere {{ incremental_predicates }}\n {%- endif %}\n {%- endset -%}\n {%- do statements.append(insert_sql) -%}\n \n {% do return(statements) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.983878, "supported_languages": null}, "macro.dbt_databricks.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) | map(attribute=\"name\") | list -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) | map(attribute=\"name\") | list -%}\n {{ insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.insert_into_sql_impl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.984077, "supported_languages": null}, "macro.dbt_databricks.insert_into_sql_impl": {"name": "insert_into_sql_impl", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.insert_into_sql_impl", "macro_sql": "{% macro insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) %}\n {%- set dest_cols_lower = dest_columns | map('lower') | list -%}\n {%- set source_cols_lower = source_columns | map('lower') | list -%}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n\n {%- if dest_cols_lower | sort == source_cols_lower | sort -%}\n {#-- All columns match (case-insensitive) --#}\n {%- if has_insert_by_name -%}\n {#-- DBR 12.2+: Use INSERT BY NAME for name-based matching --#}\n insert into {{ target_relation }} by name\n select * from {{ source_relation }}\n {%- else -%}\n {#-- DBR < 12.2: Use positional INSERT (columns must be in same order) --#}\n insert into {{ target_relation }}\n select * from {{ source_relation }}\n {%- endif -%}\n {%- else -%}\n {#-- Columns don't match, select only columns that exist in target --#}\n {#-- Note: Cannot use BY NAME with explicit column list, so use traditional syntax --#}\n {%- set common_columns = [] -%}\n {%- for dest_col in dest_columns -%}\n {%- if dest_col | lower in source_cols_lower -%}\n {%- do common_columns.append(dest_col) -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if common_columns | length > 0 -%}\n insert into {{ target_relation }} ({{ common_columns | join(', ') }})\n select {{ common_columns | join(', ') }} from {{ source_relation }}\n {%- else -%}\n {#-- No common columns, this shouldn't happen but handle it gracefully --#}\n {%- if has_insert_by_name -%}\n insert into {{ target_relation }} by name\n select * from {{ source_relation }}\n {%- else -%}\n insert into {{ target_relation }}\n select * from {{ source_relation }}\n {%- endif -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.984592, "supported_languages": null}, "macro.dbt_databricks.databricks__get_merge_sql": {"name": "databricks__get_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_merge_sql", "macro_sql": "{% macro databricks__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n\n {%- set target_alias = config.get('target_alias', 'DBT_INTERNAL_DEST') -%}\n {%- set source_alias = config.get('source_alias', 'DBT_INTERNAL_SOURCE') -%}\n\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set source_columns = (adapter.get_columns_in_relation(source) | map(attribute='name') | list)-%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set merge_with_schema_evolution = (config.get('merge_with_schema_evolution') | lower == 'true') -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set skip_matched_step = (config.get('skip_matched_step') | lower == 'true') -%}\n {%- set skip_not_matched_step = (config.get('skip_not_matched_step') | lower == 'true') -%}\n\n {%- set matched_condition = config.get('matched_condition') -%}\n {%- set not_matched_condition = config.get('not_matched_condition') -%}\n\n {%- set not_matched_by_source_action = config.get('not_matched_by_source_action') -%}\n {%- set not_matched_by_source_condition = config.get('not_matched_by_source_condition') -%}\n\n {%- set not_matched_by_source_action_trimmed = not_matched_by_source_action | lower | trim(' \\n\\t') %}\n {%- set not_matched_by_source_action_is_set = (\n not_matched_by_source_action_trimmed == 'delete'\n or not_matched_by_source_action_trimmed.startswith('update')\n )\n %}\n \n \n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n {{ source_alias }}.{{ adapter.quote(key) }} <=> {{ target_alias }}.{{ adapter.quote(key) }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n {{ source_alias }}.{{ adapter.quote(unique_key) }} <=> {{ target_alias }}.{{ adapter.quote(unique_key) }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n merge\n {%- if merge_with_schema_evolution %}\n with schema evolution\n {%- endif %}\n into\n {{ target }} as {{ target_alias }}\n using\n {{ source }} as {{ source_alias }}\n on\n {{ predicates | join('\\n and ') }}\n {%- if not skip_matched_step %}\n when matched\n {%- if matched_condition %}\n and ({{ matched_condition }})\n {%- endif %}\n then update set\n {{ get_merge_update_set(update_columns, on_schema_change, source_columns, source_alias) }}\n {%- endif %}\n {%- if not skip_not_matched_step %}\n when not matched\n {%- if not_matched_condition %}\n and ({{ not_matched_condition }})\n {%- endif %}\n then insert\n {{ get_merge_insert(on_schema_change, source_columns, source_alias) }}\n {%- endif %}\n {%- if not_matched_by_source_action_is_set %}\n when not matched by source\n {%- if not_matched_by_source_condition %}\n and ({{ not_matched_by_source_condition }})\n {%- endif %}\n then {{ not_matched_by_source_action }}\n {%- endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.incremental_validate_on_schema_change", "macro.dbt.get_merge_update_columns", "macro.dbt_databricks.get_merge_update_set", "macro.dbt_databricks.get_merge_insert"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.985947, "supported_languages": null}, "macro.dbt_databricks.get_merge_update_set": {"name": "get_merge_update_set", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_update_set", "macro_sql": "{% macro get_merge_update_set(update_columns, on_schema_change, source_columns, source_alias='DBT_INTERNAL_SOURCE') %}\n {%- if update_columns -%}\n {%- for column_name in update_columns -%}\n {{ adapter.quote(column_name) }} = {{ source_alias }}.{{ adapter.quote(column_name) }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- elif on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n {%- for column in source_columns -%}\n {{ adapter.quote(column) }} = {{ source_alias }}.{{ adapter.quote(column) }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9862661, "supported_languages": null}, "macro.dbt_databricks.get_merge_insert": {"name": "get_merge_insert", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_insert", "macro_sql": "{% macro get_merge_insert(on_schema_change, source_columns, source_alias='DBT_INTERNAL_SOURCE') %}\n {%- if on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n ({% for column in source_columns %}{{ adapter.quote(column) }}{% if not loop.last %}, {% endif %}{% endfor %}) VALUES (\n {%- for column in source_columns -%}\n {{ source_alias }}.{{ adapter.quote(column) }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %})\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.986496, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_microbatch_sql": {"name": "databricks__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_microbatch_sql", "macro_sql": "{% macro databricks__get_incremental_microbatch_sql(arg_dict) %}\n {%- set incremental_predicates = [] if arg_dict.get('incremental_predicates') is none else arg_dict.get('incremental_predicates') -%}\n {%- set event_time = model.config.event_time -%}\n {%- set start_time = config.get(\"__dbt_internal_microbatch_event_time_start\") -%}\n {%- set end_time = config.get(\"__dbt_internal_microbatch_event_time_end\") -%}\n {%- if start_time -%}\n {%- do incremental_predicates.append(\"cast(\" ~ event_time ~ \" as TIMESTAMP) >= '\" ~ start_time ~ \"'\") -%}\n {%- endif -%}\n {%- if end_time -%}\n {%- do incremental_predicates.append(\"cast(\" ~ event_time ~ \" as TIMESTAMP) < '\" ~ end_time ~ \"'\") -%}\n {%- endif -%}\n {%- do arg_dict.update({'incremental_predicates': incremental_predicates}) -%}\n {{ return(get_replace_where_sql(arg_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.986871, "supported_languages": null}, "macro.dbt_databricks.databricks__get_merge_update_columns": {"name": "databricks__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_merge_update_columns", "macro_sql": "{% macro databricks__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {# Don't quote here - columns will be quoted in get_merge_update_set #}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.column) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.987186, "supported_languages": null}, "macro.dbt_databricks.materialization_incremental_databricks": {"name": "materialization_incremental_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.materialization_incremental_databricks", "macro_sql": "{% materialization incremental, adapter='databricks', supported_languages=['sql', 'python'] -%}\n {{ log(\"MATERIALIZING INCREMENTAL\") }}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n {% set existing_relation = load_relation_with_metadata(this) %}\n {% set target_relation = this.incorporate(type='table') %}\n {% set incremental_strategy = get_incremental_strategy(catalog_relation.file_format) %}\n {% set grant_config = config.get('grants') %}\n {% set full_refresh = should_full_refresh() %}\n {% set partition_by = config.get('partition_by') %}\n {% set language = model['language'] %}\n {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}\n {% set is_delta = (catalog_relation.file_format == 'delta' and existing_relation.is_delta) %}\n {% set is_iceberg = (catalog_relation.file_format == 'iceberg' and existing_relation.is_iceberg) %}\n {% set is_replaceable_format = is_delta or is_iceberg %}\n {% set compiled_code = adapter.clean_sql(model['compiled_code']) %}\n\n {% if adapter.get_behavior_flag_no_warn('use_materialization_v2') %}\n {{ log(\"USING V2 MATERIALIZATION\") }}\n {#-- Set vars --#}\n {% set safe_create = config.get('use_safer_relation_operations', False) | as_bool %}\n {{ log(\"Safe create: \" ~ safe_create) }}\n {% set should_replace = existing_relation.is_dlt or existing_relation.is_view or full_refresh %}\n {% set is_replaceable = existing_relation.can_be_replaced and is_replaceable_format %}\n\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set staging_relation = make_staging_relation(target_relation) %}\n\n {{ run_pre_hooks() }}\n\n {% call statement('main', language=language) %}\n {{ get_create_intermediate_table(intermediate_relation, compiled_code, language) }}\n {% endcall %}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {{ log(\"No existing relation found\") }}\n {{ create_table_at(target_relation, intermediate_relation, compiled_code) }}\n {%- elif should_replace -%}\n {{ log(\"Existing relation found that requires replacement\") }}\n {% if safe_create and existing_relation.can_be_renamed %}\n {{ log(\"Safe create enabled and relation can be renamed\") }}\n {{ safe_relation_replace(existing_relation, staging_relation, intermediate_relation, compiled_code) }}\n {% else %}\n {#-- Relation must be dropped & recreated --#}\n {% if not is_replaceable %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {{ log(\"Dropping existing relation, as it is not replaceable\") }}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {{ log(\"Replacing target relation\") }}\n {{ create_table_at(target_relation, intermediate_relation, compiled_code) }}\n {% endif %}\n {%- else -%}\n {{ log(\"Existing relation found, proceeding with incremental work\")}}\n {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {{ set_overwrite_mode('DYNAMIC') }}\n {%- endif -%}\n {#-- Relation must be merged --#}\n {%- do process_schema_changes(on_schema_change, intermediate_relation, existing_relation) -%}\n {{ process_config_changes(target_relation) }}\n {% set build_sql = get_build_sql(incremental_strategy, target_relation, intermediate_relation) %}\n {%- if language == 'sql' -%}\n {#-- Check if build_sql is a list (multi-statement strategy) or a string (single statement) --#}\n {%- if build_sql is sequence and build_sql is not string -%}\n {%- for sql_statement in build_sql -%}\n {%- call statement('main') -%}\n {{ sql_statement }}\n {%- endcall -%}\n {%- endfor -%}\n {%- else -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', intermediate_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% do optimize(target_relation) %}\n\n {% if language == 'python' %}\n {{ drop_relation_if_exists(intermediate_relation) }}\n {% endif %}\n\n {{ run_post_hooks() }}\n\n {% else %}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n {% set temp_relation = make_temp_relation(target_relation) %}\n {% set incremental_predicates = config.get('predicates') or config.get('incremental_predicates') %}\n {%- set unique_key = config.get('unique_key') -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {% do apply_tags(target_relation, tags) %}\n {%- if language == 'python' -%}\n {%- do apply_tblproperties(target_relation, tblproperties) %}\n {%- endif -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- elif existing_relation.is_view or existing_relation.is_materialized_view or existing_relation.is_streaming_table or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% if not is_replaceable_format %} {#-- If Delta or Iceberg, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% if not existing_relation.is_view %}\n {% do persist_constraints(target_relation, model) %}\n {% endif %}\n {% do apply_tags(target_relation, tags) %}\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- else -%}\n {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {{ set_overwrite_mode('DYNAMIC') }}\n {%- endif -%}\n {#-- Relation must be merged --#}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set _existing_config = adapter.get_relation_config(existing_relation, model_config) -%}\n {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%}\n {%- call statement('create_temp_relation', language=language) -%}\n {{ create_table_as(True, temp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, temp_relation, existing_relation) -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': temp_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {%- set build_sql = strategy_sql_macro_func(strategy_arg_dict) -%}\n {%- if language == 'sql' -%}\n {#-- Check if build_sql is a list (multi-statement strategy) or a string (single statement) --#}\n {%- if build_sql is sequence and build_sql is not string -%}\n {%- for sql_statement in build_sql -%}\n {%- call statement('main') -%}\n {{ sql_statement }}\n {%- endcall -%}\n {%- endfor -%}\n {%- else -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', temp_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why does not either drop_relation or adapter.drop_relation work here?!\n --#}\n {%- endif -%}\n {% if _configuration_changes is not none %}\n {% set tags = _configuration_changes.changes.get(\"tags\", None) %}\n {% set tblproperties = _configuration_changes.changes.get(\"tblproperties\", None) %}\n {% set liquid_clustering = _configuration_changes.changes.get(\"liquid_clustering\") %}\n {% set row_filter = _configuration_changes.changes.get(\"row_filter\") %}\n {% set constraints = _configuration_changes.changes.get(\"constraints\") %}\n {% if tags is not none %}\n {% do apply_tags(target_relation, tags.set_tags) %}\n {%- endif -%}\n {% if tblproperties is not none %}\n {% do apply_tblproperties(target_relation, tblproperties.tblproperties) %}\n {%- endif -%}\n {% if liquid_clustering is not none %}\n {% do apply_liquid_clustered_cols(target_relation, liquid_clustering) %}\n {% endif %}\n {% if row_filter is not none %}\n {{ apply_row_filter(target_relation, row_filter) }}\n {% endif %}\n {#- Incremental constraint application requires information_schema access (see fetch_*_constraints macros) -#}\n {% set contract_config = config.get('contract') %}\n {% if constraints and contract_config and contract_config.enforced and not target_relation.is_hive_metastore() %}\n {{ apply_constraints(target_relation, constraints) }}\n {% endif %}\n {%- endif -%}\n {% do persist_docs(target_relation, model, for_relation=True) %}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n {%- endif -%}\n\n {%- if incremental_strategy == 'insert_overwrite' and not full_refresh -%}\n {{ set_overwrite_mode('STATIC') }}\n {%- endif -%}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.load_relation_with_metadata", "macro.dbt_databricks.get_incremental_strategy", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.make_intermediate_relation", "macro.dbt_databricks.make_staging_relation", "macro.dbt_databricks.run_pre_hooks", "macro.dbt.statement", "macro.dbt_databricks.get_create_intermediate_table", "macro.dbt_databricks.create_table_at", "macro.dbt_databricks.safe_relation_replace", "macro.dbt_databricks.set_overwrite_mode", "macro.dbt.process_schema_changes", "macro.dbt_databricks.process_config_changes", "macro.dbt_databricks.get_build_sql", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.optimize", "macro.dbt.drop_relation_if_exists", "macro.dbt_databricks.run_post_hooks", "macro.dbt.make_temp_relation", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt.persist_docs", "macro.dbt_databricks.apply_liquid_clustered_cols", "macro.dbt_databricks.apply_row_filter", "macro.dbt_databricks.apply_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9923232, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.set_overwrite_mode": {"name": "set_overwrite_mode", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.set_overwrite_mode", "macro_sql": "{% macro set_overwrite_mode(value) %}\n {% if adapter.is_cluster() %}\n {%- call statement('Setting partitionOverwriteMode: ' ~ value) -%}\n set spark.sql.sources.partitionOverwriteMode = {{ value }}\n {%- endcall -%}\n {% else %}\n {{ exceptions.warn(\"insert_overwrite is supported on SQL warehouses with DBR 17.1+. On older DBR versions, this strategy would be equivalent to using the table materialization.\") }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9925241, "supported_languages": null}, "macro.dbt_databricks.get_build_sql": {"name": "get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.get_build_sql", "macro_sql": "{% macro get_build_sql(incremental_strategy, target_relation, intermediate_relation) %}\n {%- set unique_key = config.get('unique_key') -%}\n {%- set incremental_predicates = config.get('predicates') or config.get('incremental_predicates') -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': intermediate_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {% do return(strategy_sql_macro_func(strategy_arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9928372, "supported_languages": null}, "macro.dbt_databricks.process_config_changes": {"name": "process_config_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.process_config_changes", "macro_sql": "{% macro process_config_changes(target_relation) %}\n {% set apply_config_changes = config.get('incremental_apply_config_changes', True) | as_bool %}\n {% if apply_config_changes %}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set existing_config = adapter.get_relation_config(target_relation, model_config) -%}\n {%- set configuration_changes = model_config.get_changeset(existing_config) -%}\n {{ apply_config_changeset(target_relation, model, configuration_changes) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.apply_config_changeset"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.993077, "supported_languages": null}, "macro.dbt_databricks.databricks__create_or_replace_clone": {"name": "databricks__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone/strategies.sql", "original_file_path": "macros/materializations/clone/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__create_or_replace_clone", "macro_sql": "{% macro databricks__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace\n table {{ this_relation.render() }}\n shallow clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.993233, "supported_languages": null}, "macro.dbt_databricks.create_or_replace_clone_external": {"name": "create_or_replace_clone_external", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone/strategies.sql", "original_file_path": "macros/materializations/clone/strategies.sql", "unique_id": "macro.dbt_databricks.create_or_replace_clone_external", "macro_sql": "{% macro create_or_replace_clone_external(this_relation, defer_relation) %}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n create or replace\n table {{ this_relation.render() }}\n shallow clone {{ defer_relation.render() }}\n {{ location_clause(catalog_relation) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.location_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.993377, "supported_languages": null}, "macro.dbt_databricks.databricks__can_clone_table": {"name": "databricks__can_clone_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone/clone.sql", "original_file_path": "macros/materializations/clone/clone.sql", "unique_id": "macro.dbt_databricks.databricks__can_clone_table", "macro_sql": "{% macro databricks__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.993664, "supported_languages": null}, "macro.dbt_databricks.materialization_clone_databricks": {"name": "materialization_clone_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone/clone.sql", "original_file_path": "macros/materializations/clone/clone.sql", "unique_id": "macro.dbt_databricks.materialization_clone_databricks", "macro_sql": "\n\n\n{%- materialization clone, adapter='databricks' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% if other_existing_relation.is_external_table %}\n {% call statement('main') %}\n {{ create_or_replace_clone_external(target_relation, defer_relation) }}\n {% endcall %}\n {% else %}\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt_databricks.create_or_replace_clone_external", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.994726, "supported_languages": ["sql"]}, "macro.dbt_databricks.databricks__get_binding_char": {"name": "databricks__get_binding_char", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__get_binding_char", "macro_sql": "{% macro databricks__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.99523, "supported_languages": null}, "macro.dbt_databricks.databricks__load_csv_rows": {"name": "databricks__load_csv_rows", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__load_csv_rows", "macro_sql": "{% macro databricks__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n {% set must_cast = model['config'].get('file_format', 'delta') == 'parquet' %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- if must_cast -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- else -%}\n {{ get_binding_char() }}\n {%- endif -%}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True, close_cursor=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.995886, "supported_languages": null}, "macro.dbt_databricks.databricks__reset_csv_table": {"name": "databricks__reset_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__reset_csv_table", "macro_sql": "{% macro databricks__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {% if old_relation.can_be_replaced and adapter.resolve_file_format(config) in ('delta', 'iceberg') %}\n {% set sql = create_or_replace_csv_table(model, agate_table, True) %}\n {% else %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {% else %}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table", "macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.996158, "supported_languages": null}, "macro.dbt_databricks.create_or_replace_csv_table": {"name": "create_or_replace_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.create_or_replace_csv_table", "macro_sql": "{% macro create_or_replace_csv_table(model, agate_table, replace=False) %}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {%- set column_comment = config.persist_column_docs() and model.columns %}\n {%- set identifier = model['alias'] -%}\n {%- set relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}\n {%- set replace_clause = \"\" -%}\n {%- if replace -%}\n {%- set replace_clause = \"or replace\" -%}\n {%- endif -%}\n\n {% set sql %}\n create {{replace_clause}} table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {%- set column_comment_clause = \"\" -%}\n {%- if column_comment and col_name in model.columns.keys() -%} \n {%- set comment = model.columns[col_name]['description'] | replace(\"'\", \"\\\\'\") -%}\n {%- if comment and comment != \"\" -%}\n {%- set column_comment_clause = \"comment '\" ~ comment ~ \"'\" -%}\n {%- endif -%}\n {%- endif -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {{ column_comment_clause }}{%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause(catalog_relation) }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(catalog_relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9971032, "supported_languages": null}, "macro.dbt_databricks.databricks__create_csv_table": {"name": "databricks__create_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__create_csv_table", "macro_sql": "{% macro databricks__create_csv_table(model, agate_table) %}\n {{ return(create_or_replace_csv_table(model, agate_table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9971871, "supported_languages": null}, "macro.dbt_databricks.log_seed_operation": {"name": "log_seed_operation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.log_seed_operation", "macro_sql": "{% macro log_seed_operation(agate_table, full_refresh_mode, create_table_sql, sql) %}\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.noop_statement", "macro.dbt.get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.99737, "supported_languages": null}, "macro.dbt_databricks.materialization_seed_databricks": {"name": "materialization_seed_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.materialization_seed_databricks", "macro_sql": "{% materialization seed, adapter='databricks' %}\n {% set target_relation = this.incorporate(type='table') %}\n\n {% if adapter.get_behavior_flag_no_warn('use_materialization_v2') %}\n {{ create_seed_v2(target_relation) }}\n {% else %}\n {{ create_seed_v1(target_relation) }}\n {% endif %}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.create_seed_v2", "macro.dbt_databricks.create_seed_v1"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9978728, "supported_languages": ["sql"]}, "macro.dbt_databricks.create_seed_v2": {"name": "create_seed_v2", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.create_seed_v2", "macro_sql": "{% macro create_seed_v2(target_relation) %}\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and (old_relation.is_view or old_relation.is_materialized_view)) -%}\n {%- set exists_as_streaming_table = (old_relation is not none and old_relation.is_streaming_table) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_pre_hooks() }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view or a materialized view\".format(old_relation)) }}\n {% elif exists_as_streaming_table %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a streaming table\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {{ log_seed_operation(agate_table, full_refresh_mode, create_table_sql, sql) }}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n -- No need to persist docs, already handled in seed create\n\n {{ run_post_hooks() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt_databricks.run_pre_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt_databricks.log_seed_operation", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.run_post_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.998687, "supported_languages": null}, "macro.dbt_databricks.create_seed_v1": {"name": "create_seed_v1", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.create_seed_v1", "macro_sql": "{% macro create_seed_v1(target_relation) %}\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and (old_relation.is_view or old_relation.is_materialized_view)) -%}\n {%- set exists_as_streaming_table = (old_relation is not none and old_relation.is_streaming_table) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view or a materialized view\".format(old_relation)) }}\n {% elif exists_as_streaming_table %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a streaming table\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n -- No need to persist docs, already handled in seed create\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9997, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_create_replace_signature_sql": {"name": "databricks__scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_create_replace_signature_sql", "macro_sql": "{% macro databricks__scalar_function_create_replace_signature_sql(target_relation) %}\n CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}})\n RETURNS {{ model.returns.data_type }}\n LANGUAGE SQL\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9999201, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_body_sql": {"name": "databricks__scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_body_sql", "macro_sql": "{% macro databricks__scalar_function_body_sql() %}\n RETURN\n {{ model.compiled_code }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.99997, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_create_replace_signature_python": {"name": "databricks__scalar_function_create_replace_signature_python", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_create_replace_signature_python", "macro_sql": "{% macro databricks__scalar_function_create_replace_signature_python(target_relation) %}\n CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql() }})\n RETURNS {{ model.returns.data_type }}\n LANGUAGE PYTHON\n AS\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0000541, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_body_python": {"name": "databricks__scalar_function_body_python", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_body_python", "macro_sql": "{% macro databricks__scalar_function_body_python() %}\n$$\n{{ model.compiled_code }}\n$$\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0001018, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_python": {"name": "databricks__scalar_function_python", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_python", "macro_sql": "{% macro databricks__scalar_function_python(target_relation) %}\n {#- Warn if user explicitly provided no-op config fields -#}\n {%- if model.config.get('runtime_version') -%}\n {{ exceptions.warn(\"'runtime_version' is accepted for compatibility but has no effect on Databricks Python UDFs. Databricks manages the Python runtime internally.\") }}\n {%- endif -%}\n {%- if model.config.get('entry_point') -%}\n {{ exceptions.warn(\"'entry_point' is accepted for compatibility but has no effect on Databricks Python UDFs. The function body is used directly.\") }}\n {%- endif -%}\n {{ databricks__scalar_function_create_replace_signature_python(target_relation) }}\n {{ databricks__scalar_function_body_python() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__scalar_function_create_replace_signature_python", "macro.dbt_databricks.databricks__scalar_function_body_python"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.000284, "supported_languages": null}, "macro.dbt_databricks.databricks__generate_database_name": {"name": "databricks__generate_database_name", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt_databricks.databricks__generate_database_name", "macro_sql": "{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}\n {%- if custom_database_name is none -%}\n {%- if node is not none and node|attr('database') -%}\n {%- set catalog_relation = adapter.build_catalog_relation(node) -%}\n {{ return(catalog_relation.catalog_name) }}\n {%- elif 'config' in target -%}\n {%- set catalog_relation = adapter.build_catalog_relation(target) -%}\n {{ return(catalog_relation.catalog_name) }}\n {%- else -%}\n {{ return(target.database) }}\n {%- endif -%}\n {%- else -%}\n {{ return(custom_database_name) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.000664, "supported_languages": null}, "macro.dbt_databricks.databricks__get_drop_sql": {"name": "databricks__get_drop_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__get_drop_sql", "macro_sql": "{% macro databricks__get_drop_sql(relation) -%}\n {%- if relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n {%- elif relation.is_streaming_table-%}\n {{ drop_streaming_table(relation) }}\n {%- elif relation.is_view or relation.is_metric_view -%}\n {{ drop_view(relation) }}\n {%- else -%}\n {{ drop_table(relation) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_materialized_view", "macro.dbt_databricks.drop_streaming_table", "macro.dbt.drop_view", "macro.dbt.drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.000881, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_relation": {"name": "databricks__drop_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_relation", "macro_sql": "{% macro databricks__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0009742, "supported_languages": null}, "macro.dbt_databricks.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {% do return(adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.001388, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_sql": {"name": "databricks__get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_sql", "macro_sql": "{% macro databricks__get_replace_sql(existing_relation, target_relation, sql) %}\n {# /* if safe_relation_replace, prefer renaming */ #}\n {% if target_relation.type == \"table\" %}\n {{ exceptions.raise_not_implemented('get_replace_sql not implemented for target of table') }}\n {% endif %}\n\n {#- Metric views always support CREATE OR REPLACE (no delta/file_format dependency) -#}\n {#- Note: existing relation is typed as VIEW from DB, so check target for metric_view -#}\n {% if target_relation.is_metric_view %}\n {{ return(get_replace_metric_view_sql(target_relation, sql)) }}\n {% endif %}\n\n {% set safe_replace = config.get('use_safer_relation_operations', False) | as_bool %}\n {% set file_format = adapter.resolve_file_format(config) %}\n {% set is_replaceable = existing_relation.type == target_relation.type and existing_relation.can_be_replaced and file_format == \"delta\" %}\n\n {% if not safe_replace %}\n {# Prioritize 'create or replace' for speed #}\n {% if is_replaceable and existing_relation.is_view %}\n {{ return(get_replace_view_sql(target_relation, sql)) }}\n {% elif is_replaceable and existing_relation.is_table %}\n {{ return(get_replace_table_sql(target_relation, sql)) }}\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ return(get_replace_materialized_view_sql(target_relation, sql)) }}\n {% endif %}\n {% endif %}\n\n {# If safe_replace, then we know that anything that would have been caught above is instead caught here #}\n {% if target_relation.can_be_renamed and existing_relation.can_be_renamed %}\n {{ return(safely_replace(existing_relation, target_relation, sql)) }}\n {% elif target_relation.can_be_renamed %}\n {{ return(stage_then_replace(existing_relation, target_relation, sql)) }}\n {% elif existing_relation.can_be_renamed %}\n {{ return(backup_and_create_in_place(existing_relation, target_relation, sql)) }}\n {% else %}\n {{ return(drop_and_create(existing_relation, target_relation, sql)) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_metric_view_sql", "macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt_databricks.safely_replace", "macro.dbt_databricks.stage_then_replace", "macro.dbt_databricks.backup_and_create_in_place", "macro.dbt_databricks.drop_and_create"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.002152, "supported_languages": null}, "macro.dbt_databricks.safely_replace": {"name": "safely_replace", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.safely_replace", "macro_sql": "{% macro safely_replace(existing_relation, target_relation, sql) %}\n {{ log('Using safely_replace') }}\n {% set staging_relation = make_staging_relation(target_relation, type='view') %}\n {{ drop_relation_if_exists(staging_relation) }}\n {% call statement(name=\"main\") %}\n {{ get_create_sql(staging_relation, sql) }}\n {% endcall %}\n {{ create_backup(existing_relation) }}\n {{ return([\n get_rename_sql(staging_relation, existing_relation.render()),\n get_drop_backup_sql(existing_relation)\n ]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.make_staging_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt_databricks.create_backup", "macro.dbt.get_rename_sql", "macro.dbt.get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.002402, "supported_languages": null}, "macro.dbt_databricks.stage_then_replace": {"name": "stage_then_replace", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.stage_then_replace", "macro_sql": "{% macro stage_then_replace(existing_relation, target_relation, sql) %}\n {{ log('Using stage_then_replace') }}\n {% set staging_relation = make_staging_relation(target_relation, type='view') %}\n {{ drop_relation_if_exists(staging_relation) }}\n {% call statement(name=\"main\") %}\n {{ get_create_sql(staging_relation, sql) }}\n {% endcall %}\n\n {{ return([\n get_drop_sql(existing_relation),\n get_rename_sql(staging_relation, existing_relation.render()),\n ]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.make_staging_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0026188, "supported_languages": null}, "macro.dbt_databricks.backup_and_create_in_place": {"name": "backup_and_create_in_place", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.backup_and_create_in_place", "macro_sql": "{% macro backup_and_create_in_place(existing_relation, target_relation, sql) %}\n {{ log('Using backup_and_create_in_place') }}\n {{ create_backup(existing_relation) }}\n {{ return([\n get_create_sql(target_relation, sql),\n get_drop_backup_sql(existing_relation)\n ]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_backup", "macro.dbt.get_create_sql", "macro.dbt.get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.00275, "supported_languages": null}, "macro.dbt_databricks.drop_and_create": {"name": "drop_and_create", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.drop_and_create", "macro_sql": "{% macro drop_and_create(existing_relation, target_relation, sql) %}\n {{ log('Using drop_and_create') }}\n {{ return([\n get_drop_sql(existing_relation),\n get_create_sql(target_relation, sql)\n ]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.002859, "supported_languages": null}, "macro.dbt_databricks.databricks__comment_clause": {"name": "databricks__comment_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/comment.sql", "original_file_path": "macros/relations/comment.sql", "unique_id": "macro.dbt_databricks.databricks__comment_clause", "macro_sql": "{% macro databricks__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation and model.description -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {%- endif -%}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0031428, "supported_languages": null}, "macro.dbt_databricks.fetch_tags": {"name": "fetch_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags", "macro_sql": "{% macro fetch_tags(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {% call statement('list_tags', fetch_result=True) -%}\n {{ fetch_tags_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_tags').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_tags_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.003464, "supported_languages": null}, "macro.dbt_databricks.fetch_tags_sql": {"name": "fetch_tags_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags_sql", "macro_sql": "{% macro fetch_tags_sql(relation) -%}\n SELECT tag_name, tag_value\n FROM `system`.`information_schema`.`table_tags`\n WHERE catalog_name = '{{ relation.database|lower }}' \n AND schema_name = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0035589, "supported_languages": null}, "macro.dbt_databricks.apply_tags": {"name": "apply_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.apply_tags", "macro_sql": "{% macro apply_tags(relation, set_tags) -%}\n {{ log(\"Applying tags to relation \" ~ set_tags) }}\n {%- if set_tags and relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif -%}\n {%- if set_tags and set_tags != [] %}\n {%- call statement('main') -%}\n {{ alter_set_tags(relation, set_tags) }}\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_set_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0037482, "supported_languages": null}, "macro.dbt_databricks.alter_set_tags": {"name": "alter_set_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_set_tags", "macro_sql": "{% macro alter_set_tags(relation, tags) -%}\n ALTER {{ relation.type.render_for_alter() }} {{ relation.render() }} SET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' = '{{ tags[tag] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0039198, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_intermediate_sql": {"name": "databricks__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_intermediate_sql", "macro_sql": "{%- macro databricks__get_create_intermediate_sql(relation, sql) -%}\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ drop_relation(intermediate_relation) }}\n\n {{ return(get_create_sql(intermediate_relation, sql)) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.drop_relation", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.004071, "supported_languages": null}, "macro.dbt_databricks.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/location.sql", "original_file_path": "macros/relations/location.sql", "unique_id": "macro.dbt_databricks.location_clause", "macro_sql": "{% macro location_clause(relation) %}\n {#--\n Moving forward, `relation` should be a `CatalogRelation`, which is covered by the first condition.\n However, there could be existing macros that are still passing in a `BaseRelation`, including user macros.\n Hence, we need to support the old code still, which is covered by the second condition.\n --#}\n {%- if relation.catalog_type is not none -%}\n\n {%- if relation.location is not none -%}\n location '{{ relation.location }}{% if is_incremental() %}_tmp{% endif %}'\n {%- endif -%}\n\n {%- else -%}\n\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n {%- set model_path = adapter.compute_external_path(config, model, is_incremental()) %}\n location '{{ model_path }}'\n {%- elif (not relation.is_hive_metastore()) and file_format != 'delta' -%}\n {{ exceptions.raise_compiler_error(\n 'Incompatible configuration: `location_root` must be set when using a non-delta file format with Unity Catalog'\n ) }}\n {%- endif %}\n\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.is_incremental"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.004601, "supported_languages": null}, "macro.dbt_databricks.optimize": {"name": "optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.optimize", "macro_sql": "{% macro optimize(relation) %}\n {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.004975, "supported_languages": null}, "macro.dbt_databricks.databricks__optimize": {"name": "databricks__optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.databricks__optimize", "macro_sql": "\n\n{%- macro databricks__optimize(relation) -%}\n {%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and\n var('databricks_skip_optimize', 'false')|lower != 'true' and\n adapter.resolve_file_format(config) == 'delta' -%}\n {%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) or config.get('auto_liquid_cluster', False) -%}\n {%- call statement('run_optimize_stmt') -%}\n {{ get_optimize_sql(relation) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_optimize_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.005306, "supported_languages": null}, "macro.dbt_databricks.get_optimize_sql": {"name": "get_optimize_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.get_optimize_sql", "macro_sql": "{%- macro get_optimize_sql(relation) %}\n optimize {{ relation.render() }}\n {%- if config.get('zorder', False) and adapter.resolve_file_format(config) == 'delta' %}\n {%- if config.get('liquid_clustered_by', False) or config.get('auto_liquid_cluster', False) %}\n {{ exceptions.warn(\"Both zorder and liquid_clustering are set but they are incompatible. zorder will be ignored.\") }}\n {%- else %}\n {%- set zorder = config.get('zorder', none) %}\n {# TODO: predicates here? WHERE ... #}\n {%- if zorder is sequence and zorder is not string %}\n zorder by (\n {%- for col in zorder %}\n {{ col }}{% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- else %}\n zorder by ({{zorder}})\n {%- endif %}\n {%- endif %}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0058408, "supported_languages": null}, "macro.dbt_databricks.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/file_format.sql", "original_file_path": "macros/relations/file_format.sql", "unique_id": "macro.dbt_databricks.file_format_clause", "macro_sql": "{% macro file_format_clause(catalog_relation=none) %}\n {#--\n Moving forward, this macro should require a `catalog_relation`, which is covered by the first condition.\n However, there could be existing macros that is still passing no arguments, including user macros.\n Hence, we need to support the old code still, which is covered by the second condition.\n --#}\n {% if catalog_relation is not none %}\n {%- set table_format = catalog_relation.table_format -%}\n {%- set file_format = catalog_relation.file_format -%}\n {% else %}\n {%- set table_format = config.get('table_format', default='default') -%}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n {% endif %}\n \n {#-- Use managed Iceberg if behavior flag is enabled and table_format is iceberg --#}\n {% if table_format == 'iceberg' and adapter.behavior.use_managed_iceberg %}\n using iceberg\n {% else %}\n using {{ file_format }}\n {% endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.006356, "supported_languages": null}, "macro.dbt_databricks.get_file_format": {"name": "get_file_format", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/file_format.sql", "original_file_path": "macros/relations/file_format.sql", "unique_id": "macro.dbt_databricks.get_file_format", "macro_sql": "{% macro get_file_format(catalog_relation=none) %}\n {#-\n Moving forward, this macro should require a `catalog_relation`, which is covered by the first condition.\n However, there could be existing macros that is still passing no arguments, including user macros.\n Hence, we need to support the old code still, which is covered by the second condition.\n -#}\n {% if catalog_relation is not none %}\n {%- set raw_file_format = catalog_relation.file_format -%}\n {% else %}\n {%- set raw_file_format = adapter.resolve_file_format(config) -%}\n {% endif %}\n {% do return(dbt_databricks_validate_get_file_format(raw_file_format)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.dbt_databricks_validate_get_file_format"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.006557, "supported_languages": null}, "macro.dbt_databricks.liquid_clustered_cols": {"name": "liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.liquid_clustered_cols", "macro_sql": "{% macro liquid_clustered_cols() -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set auto_cluster = config.get('auto_liquid_cluster', validator=validation.any[boolean]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n CLUSTER BY ({{ cols | join(', ') }})\n {%- elif auto_cluster -%}\n CLUSTER BY AUTO\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0070348, "supported_languages": null}, "macro.dbt_databricks.apply_liquid_clustered_cols": {"name": "apply_liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.apply_liquid_clustered_cols", "macro_sql": "{% macro apply_liquid_clustered_cols(target_relation, liquid_clustering) -%}\n {%- set cols = liquid_clustering.cluster_by -%}\n {%- set auto_cluster = liquid_clustering.auto_cluster -%}\n {%- if cols and cols != [] %}\n {%- call statement('set_cluster_by_columns') -%}\n ALTER {{ target_relation.type.render() }} {{ target_relation.render() }} CLUSTER BY ({{ cols | join(', ') }})\n {%- endcall -%}\n {%- elif auto_cluster -%}\n {%- call statement('set_cluster_by_auto') -%}\n ALTER {{ target_relation.type.render() }} {{ target_relation.render() }} CLUSTER BY AUTO\n {%- endcall -%}\n {% else %}\n {%- call statement('unset_cluster_by') -%}\n ALTER {{ target_relation.type.render() }} {{ target_relation.render() }} CLUSTER BY NONE\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.007434, "supported_languages": null}, "macro.dbt_databricks.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() -%}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.007676, "supported_languages": null}, "macro.dbt_databricks.databricks__tblproperties_clause": {"name": "databricks__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.databricks__tblproperties_clause", "macro_sql": "{% macro databricks__tblproperties_clause(tblproperties=None) -%}\n {%- if adapter.is_uniform(config) -%}\n {%- set tblproperties = adapter.update_tblproperties_for_uniform_iceberg(config, tblproperties) -%}\n {%- else -%}\n {%- set tblproperties = tblproperties or config.get(\"tblproperties\", {}) -%}\n {%- endif -%}\n {%- if tblproperties != {} %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.008002, "supported_languages": null}, "macro.dbt_databricks.apply_tblproperties": {"name": "apply_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.apply_tblproperties", "macro_sql": "{% macro apply_tblproperties(relation, tblproperties) -%}\n {% set tblproperty_statment = databricks__tblproperties_clause(tblproperties) %}\n {% if tblproperty_statment %}\n {%- call statement('main') -%}\n ALTER {{ relation.type.render_for_alter() }} {{ relation.render() }} SET {{ tblproperty_statment}}\n {%- endcall -%}\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.008196, "supported_languages": null}, "macro.dbt_databricks.execute_no_op": {"name": "execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/execute.sql", "original_file_path": "macros/relations/execute.sql", "unique_id": "macro.dbt_databricks.execute_no_op", "macro_sql": "{% macro execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.008372, "supported_languages": null}, "macro.dbt_databricks.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.009697, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_constraints": {"name": "databricks__persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__persist_constraints", "macro_sql": "{% macro databricks__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% set has_model_contract = contract_config and contract_config.enforced %}\n {% set has_databricks_constraints = config.get('persist_constraints', False) %}\n\n {% if (has_model_contract or has_databricks_constraints) %}\n {% if adapter.resolve_file_format(config) != 'delta' %}\n {# Constraints are only supported for delta tables #}\n {{ exceptions.warn(\"Constraints not supported for file format: \" ~ adapter.resolve_file_format(config)) }}\n {% elif relation.is_view %}\n {# Constraints are not supported for views. This point in the code should not have been reached. #}\n {{ exceptions.raise_compiler_error(\"Constraints not supported for views.\") }}\n {% elif is_incremental() %}\n {# Constraints are not applied for incremental updates. This point in the code should not have been reached #}\n {{ exceptions.raise_compiler_error(\"Constraints are not applied for incremental updates. Full refresh is required to update constraints.\") }}\n {% else %}\n {% do alter_column_set_constraints(relation, model) %}\n {% do alter_table_add_constraints(relation, model) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt_databricks.alter_column_set_constraints", "macro.dbt_databricks.alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010078, "supported_languages": null}, "macro.dbt_databricks.apply_alter_constraints": {"name": "apply_alter_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.apply_alter_constraints", "macro_sql": "{% macro apply_alter_constraints(relation) %}\n {%- for constraint in relation.alter_constraints -%}\n {% call statement('add constraint') %}\n ALTER TABLE {{ relation.render() }} ADD {{ constraint.render() }}\n {% endcall %}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010201, "supported_languages": null}, "macro.dbt_databricks.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010293, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_table_add_constraints": {"name": "databricks__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_table_add_constraints", "macro_sql": "{% macro databricks__alter_table_add_constraints(relation, model) %}\n {% set constraints = get_model_constraints(model) %}\n {% set statements = get_constraints_sql(relation, constraints, model) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_model_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010447, "supported_languages": null}, "macro.dbt_databricks.get_model_constraints": {"name": "get_model_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_model_constraints", "macro_sql": "{% macro get_model_constraints(model) %}\n {% set constraints = model.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and model.get('meta', {}).get('constraints') is sequence %}\n {# Databricks constraints implementation. Constraints are in the meta property. #}\n {% set db_constraints = model.get('meta', {}).get('constraints', []) %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010711, "supported_languages": null}, "macro.dbt_databricks.get_column_constraints": {"name": "get_column_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_column_constraints", "macro_sql": "{% macro get_column_constraints(column) %}\n {% set constraints = column.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and column.get('meta', {}).get('constraint') %}\n {# Databricks constraints implementation. Constraint is in the meta property. #}\n {% set db_constraints = [column.get('meta', {}).get('constraint')] %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints, column) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010987, "supported_languages": null}, "macro.dbt_databricks.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0110788, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_set_constraints": {"name": "databricks__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_set_constraints", "macro_sql": "{% macro databricks__alter_column_set_constraints(relation, model) %}\n {% set column_dict = model.columns %}\n {% for column_name in column_dict %}\n {% set column = column_dict[column_name] %}\n {% set constraints = get_column_constraints(column) %}\n {% set statements = get_constraints_sql(relation, constraints, model, column) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0113149, "supported_languages": null}, "macro.dbt_databricks.get_constraints_sql": {"name": "get_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraints_sql", "macro_sql": "{% macro get_constraints_sql(relation, constraints, model, column={}) %}\n {% set statements = [] %}\n -- Hack so that not null constraints will be applied before other constraints\n {% for constraint in constraints|selectattr('type', 'eq', 'not_null') %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endfor %}\n {% for constraint in constraints|rejectattr('type', 'eq', 'not_null') %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endfor %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_constraint_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0118499, "supported_languages": null}, "macro.dbt_databricks.get_constraint_sql": {"name": "get_constraint_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraint_sql", "macro_sql": "{% macro get_constraint_sql(relation, constraint, model, column={}) %}\n {% set statements = [] %}\n {% set type = constraint.get('type', '') %}\n\n {% if type == 'check' %}\n {% set expression = constraint.get('expression', '') %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Invalid check constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get('name') %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5 (relation.identifier ~ \";\" ~ column.get('name', '') ~ \";\" ~ expression ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" check (\" ~ expression ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'not_null' %}\n {% set column_names = constraint.get('columns', []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if column %}\n {% set quoted_name = adapter.quote(column['name']) %}\n {% set stmt = \"alter table \" ~ relation.render() ~ \" change column \" ~ quoted_name ~ \" set not null \" ~ (constraint.expression or \"\") ~ \";\" %}\n {% do statements.append(stmt) %}\n {% else %}\n {{ exceptions.warn('not_null constraint on invalid column: ' ~ column_name) }}\n {% endif %}\n {% endfor %}\n {% elif type == 'primary_key' %}\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ type)}}\n {% endif %}\n {% set column_names = constraint.get('columns', []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid primary key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set name = constraint.get('name') %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"primary_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" primary key(\" ~ joined_names ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'foreign_key' %}\n\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {% set name = constraint.get('name') %}\n \n {% if constraint.get('expression') %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ constraint.get('expression') ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" foreign key\" ~ constraint.get('expression') %}\n {% else %}\n {% set column_names = constraint.get('columns', []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid foreign key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set parent = constraint.get('to') %}\n {% if not parent %}\n {{ exceptions.raise_compiler_error('No parent table defined for foreign key: ' ~ expression) }}\n {% endif %}\n {% if not \".\" in parent %}\n {% set parent_relation = api.Relation.create(database=relation.database, schema=relation.schema, identifier=parent, type='table') %}\n {% set parent = parent_relation.render() %}\n {% endif %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\" ~ parent ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" foreign key(\" ~ joined_names ~ \") references \" ~ parent %}\n {% set parent_columns = constraint.get('to_columns') %}\n {% if parent_columns %}\n {% set quoted_parent_columns = [] %}\n {% for parent_column in parent_columns %}\n {% do quoted_parent_columns.append(adapter.quote(parent_column)) %}\n {% endfor %}\n {% set stmt = stmt ~ \"(\" ~ quoted_parent_columns|join(\", \") ~ \")\"%}\n {% endif %}\n {% endif %}\n {% set stmt = stmt ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'custom' %}\n {% set expression = constraint.get('expression', '') %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Missing custom constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get('name') %}\n {% set expression = constraint.get('expression') %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5 (relation.identifier ~ \";\" ~ expression ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" \" ~ expression ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif constraint.get('warn_unsupported') %}\n {{ exceptions.warn(\"unsupported constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.01566, "supported_languages": null}, "macro.dbt_databricks.databricks_constraints_to_dbt": {"name": "databricks_constraints_to_dbt", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks_constraints_to_dbt", "macro_sql": "{% macro databricks_constraints_to_dbt(constraints, column) %}\n {# convert constraints defined using the original databricks format #}\n {% set dbt_constraints = [] %}\n {% for constraint in constraints %}\n {% if constraint.get and constraint.get('type') %}\n {# already in model contract format #}\n {% do dbt_constraints.append(constraint) %}\n {% else %}\n {% if column %}\n {% if constraint == \"not_null\" %}\n {% do dbt_constraints.append({\"type\": \"not_null\", \"columns\": [column.get('name')]}) %}\n {% else %}\n {{ exceptions.raise_compiler_error('Invalid constraint for column ' ~ column.get('name', \"\") ~ '. Only `not_null` is supported.') }}\n {% endif %}\n {% else %}\n {% set name = constraint['name'] %}\n {% if not name %}\n {{ exceptions.raise_compiler_error('Invalid check constraint name') }}\n {% endif %}\n {% set condition = constraint['condition'] %}\n {% if not condition %}\n {{ exceptions.raise_compiler_error('Invalid check constraint condition') }}\n {% endif %}\n {% do dbt_constraints.append({\"name\": name, \"type\": \"check\", \"expression\": condition}) %}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(dbt_constraints) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0166001, "supported_languages": null}, "macro.dbt_databricks.create_backup": {"name": "create_backup", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt_databricks.create_backup", "macro_sql": "{%- macro create_backup(relation) -%}\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ adapter.rename_relation(relation, backup_relation) }}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.016763, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_sql": {"name": "databricks__get_create_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_sql", "macro_sql": "{%- macro databricks__get_create_sql(relation, sql) -%}\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_streaming_table -%}\n {{ get_create_streaming_table_as_sql(relation, sql) }}\n\n {%- elif relation.is_metric_view -%}\n {{ get_create_metric_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro.dbt_databricks.get_create_metric_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.017077, "supported_languages": null}, "macro.dbt_databricks.get_configuration_changes": {"name": "get_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/config.sql", "original_file_path": "macros/relations/config.sql", "unique_id": "macro.dbt_databricks.get_configuration_changes", "macro_sql": "{%- macro get_configuration_changes(existing_relation) -%}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set existing_config = adapter.get_relation_config(existing_relation, model_config) -%}\n {%- set configuration_changes = model_config.get_changeset(existing_config) -%}\n {% do return(configuration_changes) %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.017251, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_materialized_view": {"name": "databricks__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_materialized_view", "macro_sql": "{% macro databricks__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.017326, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_materialized_view_sql": {"name": "databricks__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_materialized_view_sql", "macro_sql": "{% macro databricks__get_replace_materialized_view_sql(target_relation, sql) %}\n {{ get_create_materialized_view_as_sql(target_relation, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0174122, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_materialized_view": {"name": "databricks__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_materialized_view", "macro_sql": "{% macro databricks__refresh_materialized_view(relation) -%}\n refresh materialized view {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0174851, "supported_languages": null}, "macro.dbt_databricks.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.017803, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql": {"name": "databricks__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {#- CREATE OR REPLACE cannot change partition_by, so use DROP + CREATE when partition_by changes -#}\n {% if configuration_changes.changes[\"partition_by\"] %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {% do return(drop_and_create(existing_relation, relation, sql)) %}\n {% else %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n {% endif %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {%- set alter_statement = get_alter_mv_internal(relation, configuration_changes) -%}\n {%- set return_statements = [] -%}\n {%- if alter_statement -%}\n {{ return_statements.append(alter_statement) }}\n {%- endif -%}\n {%- set tags = configuration_changes.changes[\"tags\"] -%}\n {%- if tags and tags.set_tags and tags.set_tags != [] -%}\n {{ return_statements.append(alter_set_tags(relation, tags.set_tags)) }}\n {%- endif -%}\n\n {#- Row filter handling - append SQL to list, don't execute -#}\n {#- is_change guard prevents false alters when row_filter is unchanged -#}\n {%- set row_filter = configuration_changes.changes.get(\"row_filter\") -%}\n {%- if row_filter and row_filter.is_change -%}\n {%- if row_filter.should_unset -%}\n {{ return_statements.append(alter_drop_row_filter(relation)) }}\n {%- elif row_filter.function -%}\n {{ return_statements.append(alter_set_row_filter(relation, row_filter)) }}\n {%- endif -%}\n {%- endif -%}\n\n {% do return(return_statements) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.drop_and_create", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_mv_internal", "macro.dbt_databricks.alter_set_tags", "macro.dbt_databricks.alter_drop_row_filter", "macro.dbt_databricks.alter_set_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.01838, "supported_languages": null}, "macro.dbt_databricks.get_alter_mv_internal": {"name": "get_alter_mv_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_mv_internal", "macro_sql": "{% macro get_alter_mv_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n {%- if refresh -%}\n -- Currently only schedule can be altered\n ALTER MATERIALIZED VIEW {{ relation.render() }}\n {{ get_alter_sql_refresh_schedule(refresh) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.018514, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql": {"name": "databricks__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_create_materialized_view_as_sql(relation, sql) -%}\n {# Column masks are supported in DBSQL, but not yet wired up to the adapter. Return a helpful error until supported. #}\n {% if column_mask_exists() %}\n {% do exceptions.raise_compiler_error(\"Column masks are not yet supported for materialized views.\") %}\n {% endif %}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = materialized_view.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = materialized_view.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = materialized_view.config[\"comment\"].comment -%}\n {%- set refresh = materialized_view.config[\"refresh\"] -%}\n\n {#\n TODO: When DESCRIBE QUERY EXTENDED is supported, this implementation should be simplified\n to use that instead. For now, we work around this limitation by writing results to a\n temporary view and using DESCRIBE TABLE EXTENDED on the temporary view.\n #}\n {%- set temp_relation = make_temp_relation(relation) -%}\n {% call statement('create_temp_view') -%}\n {%- set sql_with_limit = sql.rstrip('; \\n\\t') ~ ' LIMIT 10' -%}\n {{ create_temporary_view(temp_relation, sql_with_limit) }}\n {%- endcall %}\n\n {%- set columns = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set model_columns = model.get('columns', {}) -%}\n {%- set contract_config = config.get('contract') -%}\n {%- set contract_enforced = contract_config and contract_config.enforced -%}\n {%- if contract_enforced -%}\n {%- do exceptions.warn(\n \"contract.enforced=true on materialized_view '\" ~ model.name ~ \"': not supported by dbt (https://docs.getdbt.com/docs/mesh/govern/model-contracts). dbt-databricks provides best-effort support that may change without notice.\"\n ) -%}\n {%- set model_constraints = model.get('constraints', []) -%}\n {%- else -%}\n {%- set model_constraints = [] -%}\n {%- endif -%}\n {%- set columns_and_constraints = adapter.parse_columns_and_constraints(columns, model_columns, model_constraints, contract_enforced, model.name) -%}\n {%- set target_relation = relation.enrich(columns_and_constraints[1]) -%}\n\n create or replace materialized view {{ target_relation.render() }}\n {{ get_column_and_constraints_sql(target_relation, columns_and_constraints[0]) }}\n {{ get_create_row_filter_clause(target_relation) }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ liquid_clustered_cols() }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh) }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.column_mask_exists", "macro.dbt.make_temp_relation", "macro.dbt.statement", "macro.dbt_spark.create_temporary_view", "macro.dbt_databricks.get_column_and_constraints_sql", "macro.dbt_databricks.get_create_row_filter_clause", "macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.019477, "supported_languages": null}, "macro.dbt_databricks.drop_streaming_table": {"name": "drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.drop_streaming_table", "macro_sql": "{% macro drop_streaming_table(relation) -%}\n {{ return(adapter.dispatch('drop_streaming_table', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.default__drop_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.019594, "supported_languages": null}, "macro.dbt_databricks.default__drop_streaming_table": {"name": "default__drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.default__drop_streaming_table", "macro_sql": "{% macro default__drop_streaming_table(relation) -%}\n drop table if exists {{ relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0196512, "supported_languages": null}, "macro.dbt_databricks.refresh_streaming_table": {"name": "refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.refresh_streaming_table", "macro_sql": "{% macro refresh_streaming_table(relation, sql) -%}\n {{ adapter.dispatch('refresh_streaming_table', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.019776, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_streaming_table": {"name": "databricks__refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_streaming_table", "macro_sql": "{% macro databricks__refresh_streaming_table(relation, sql) -%}\n refresh streaming table {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0198421, "supported_languages": null}, "macro.dbt_databricks.get_alter_streaming_table_as_sql": {"name": "get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_streaming_table_as_sql", "macro_sql": "{% macro get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_streaming_table_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.020233, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql": {"name": "databricks__get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {%- set alter_statement = get_alter_st_internal(relation, configuration_changes) -%}\n {%- set create_statement = get_create_st_internal(relation, configuration_changes, sql) -%}\n {%- set return_statements = [] -%}\n {%- if create_statement -%}\n {{ return_statements.append(create_statement) }}\n {%- endif -%}\n {%- if alter_statement -%}\n {{ return_statements.append(alter_statement) }}\n {%- endif -%}\n {%- set tags = configuration_changes.changes[\"tags\"] -%}\n {%- if tags and tags.set_tags and tags.set_tags != [] -%}\n {{ return_statements.append(alter_set_tags(relation, tags.set_tags)) }}\n {%- endif -%}\n\n {#- Row filter handling - append SQL to list, don't execute -#}\n {#- is_change guard prevents false alters from `diff or value` fallback in streaming_table.py:56 -#}\n {%- set row_filter = configuration_changes.changes.get(\"row_filter\") -%}\n {%- if row_filter and row_filter.is_change -%}\n {%- if row_filter.should_unset -%}\n {{ return_statements.append(alter_drop_row_filter(relation)) }}\n {%- elif row_filter.function -%}\n {{ return_statements.append(alter_set_row_filter(relation, row_filter)) }}\n {%- endif -%}\n {%- endif -%}\n\n {% do return(return_statements) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_st_internal", "macro.dbt_databricks.get_create_st_internal", "macro.dbt_databricks.alter_set_tags", "macro.dbt_databricks.alter_drop_row_filter", "macro.dbt_databricks.alter_set_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.020809, "supported_languages": null}, "macro.dbt_databricks.get_create_st_internal": {"name": "get_create_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_create_st_internal", "macro_sql": "{% macro get_create_st_internal(relation, configuration_changes, sql) %}\n {%- set partition_by = configuration_changes.changes[\"partition_by\"].partition_by -%}\n {%- set tblproperties = configuration_changes.changes[\"tblproperties\"].tblproperties -%}\n {%- set comment = configuration_changes.changes[\"comment\"].comment -%}\n CREATE OR REFRESH STREAMING TABLE {{ relation.render() }}\n {% if partition_by -%}\n {{ get_create_sql_partition_by(partition_by) }}\n {%- endif %}\n {{ liquid_clustered_cols() }}\n {% if comment -%}\n {{ get_create_sql_comment(comment) }}\n {%- endif %}\n {% if tblproperties -%}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {%- endif %}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.021146, "supported_languages": null}, "macro.dbt_databricks.get_alter_st_internal": {"name": "get_alter_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_st_internal", "macro_sql": "{% macro get_alter_st_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n {%- set is_scheduled = refresh and (refresh.cron or refresh.every or refresh.on_update) -%}\n {%- if is_scheduled -%}\n ALTER STREAMING TABLE {{ relation.render() }}\n ADD {{ get_create_sql_refresh_schedule(refresh) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.021329, "supported_languages": null}, "macro.dbt_databricks.get_create_streaming_table_as_sql": {"name": "get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro_sql": "{% macro get_create_streaming_table_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_streaming_table_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.021562, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql": {"name": "databricks__get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_create_streaming_table_as_sql(relation, sql) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = streaming_table.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = streaming_table.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = streaming_table.config[\"comment\"].comment -%}\n {%- set refresh = streaming_table.config[\"refresh\"] -%}\n\n {%- set analysis_sql = sql | replace('STREAM ', '') | replace('stream ', '') -%}\n\n {#\n TODO: When DESCRIBE QUERY EXTENDED is supported, this implementation should be simplified\n to use that instead. For now, we work around this limitation by writing results to a\n temporary view and using DESCRIBE TABLE EXTENDED on the temporary view.\n #}\n {%- set temp_relation = make_temp_relation(relation) -%}\n {% call statement('create_temp_view') -%}\n {%- set view_sql = analysis_sql.rstrip('; \\n\\t') -%}\n {{ create_temporary_view(temp_relation, view_sql) }}\n {%- endcall %}\n\n {%- set columns = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set model_columns = model.get('columns', {}) -%}\n {%- set contract_config = config.get('contract') -%}\n {%- set contract_enforced = contract_config and contract_config.enforced -%}\n {%- set columns_and_constraints = adapter.parse_columns_and_constraints(columns, model_columns, [], contract_enforced, model.name) -%}\n\n {#-- We don't enrich the relation with model constraints because they are not supported for streaming tables --#}\n CREATE STREAMING TABLE {{ relation.render() }}\n {{ get_column_and_constraints_sql(relation, columns_and_constraints[0]) }}\n {{ get_create_row_filter_clause(relation) }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ liquid_clustered_cols() }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh) }}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.statement", "macro.dbt_spark.create_temporary_view", "macro.dbt_databricks.get_column_and_constraints_sql", "macro.dbt_databricks.get_create_row_filter_clause", "macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.022214, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_comment": {"name": "get_create_sql_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/comment.sql", "original_file_path": "macros/relations/components/comment.sql", "unique_id": "macro.dbt_databricks.get_create_sql_comment", "macro_sql": "{%- macro get_create_sql_comment(comment) -%}\n{% if comment is string -%}\n COMMENT '{{ comment }}'\n{%- endif -%}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.022325, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_refresh_schedule": {"name": "get_create_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_create_sql_refresh_schedule", "macro_sql": "{% macro get_create_sql_refresh_schedule(refresh) %}\n {%- if refresh.cron -%}\n SCHEDULE CRON '{{ refresh.cron }}'{%- if refresh.time_zone_value %} AT TIME ZONE '{{ refresh.time_zone_value }}'{%- endif -%}\n {%- elif refresh.every -%}\n SCHEDULE EVERY {{ refresh.every }}\n {%- elif refresh.on_update -%}\n TRIGGER ON UPDATE{%- if refresh.at_most_every %} AT MOST EVERY INTERVAL {{ refresh.at_most_every }}{%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.022584, "supported_languages": null}, "macro.dbt_databricks.get_alter_sql_refresh_schedule": {"name": "get_alter_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_alter_sql_refresh_schedule", "macro_sql": "{% macro get_alter_sql_refresh_schedule(refresh) %}\n {%- if not (refresh.cron or refresh.every or refresh.on_update) -%}\n DROP SCHEDULE\n {%- else -%}\n {{- 'ALTER ' if refresh.is_altered else 'ADD ' -}}{{- get_create_sql_refresh_schedule(refresh) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.022727, "supported_languages": null}, "macro.dbt_databricks.alter_query": {"name": "alter_query", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/query.sql", "original_file_path": "macros/relations/components/query.sql", "unique_id": "macro.dbt_databricks.alter_query", "macro_sql": "{% macro alter_query(target_relation, query) %}\n {{ log(\"Altering query\") }}\n {% if query %}\n {% call statement('main') %}\n {{- get_alter_query_sql(target_relation, query) }}\n {% endcall %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_alter_query_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0229278, "supported_languages": null}, "macro.dbt_databricks.get_alter_query_sql": {"name": "get_alter_query_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/query.sql", "original_file_path": "macros/relations/components/query.sql", "unique_id": "macro.dbt_databricks.get_alter_query_sql", "macro_sql": "{% macro get_alter_query_sql(target_relation, query) -%}\n ALTER {{ target_relation.type.render_for_alter() }} {{ target_relation.render() }} AS (\n {{ query }}\n )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.023026, "supported_languages": null}, "macro.dbt_databricks.fetch_row_filters": {"name": "fetch_row_filters", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.fetch_row_filters", "macro_sql": "\n\n{%- macro fetch_row_filters(relation) -%}\n {%- if relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Row filters are not supported for Hive Metastore\") }}\n {%- endif -%}\n {%- call statement('list_row_filters', fetch_result=True) -%}\n {{ fetch_row_filters_sql(relation) }}\n {%- endcall -%}\n {%- do return(load_result('list_row_filters').table) -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_row_filters_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.023744, "supported_languages": null}, "macro.dbt_databricks.fetch_row_filters_sql": {"name": "fetch_row_filters_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.fetch_row_filters_sql", "macro_sql": "{%- macro fetch_row_filters_sql(relation) -%}\n SELECT\n table_catalog,\n table_schema,\n table_name,\n filter_name,\n target_columns\n FROM `{{ relation.database }}`.`information_schema`.`row_filters`\n WHERE table_catalog = '{{ relation.database | lower }}'\n AND table_schema = '{{ relation.schema | lower }}'\n AND table_name = '{{ relation.identifier | lower }}'\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.023864, "supported_languages": null}, "macro.dbt_databricks.quote_row_filter_columns": {"name": "quote_row_filter_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.quote_row_filter_columns", "macro_sql": "\n\n{%- macro quote_row_filter_columns(columns) -%}\n {#- Quote column names with backticks and join with comma.\n Shared by alter_set_row_filter and get_create_row_filter_clause. -#}\n {%- set quoted = [] -%}\n {%- for col in columns -%}\n {%- do quoted.append('`' ~ col ~ '`') -%}\n {%- endfor -%}\n {{- quoted | join(', ') -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0240052, "supported_languages": null}, "macro.dbt_databricks.quote_row_filter_function": {"name": "quote_row_filter_function", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.quote_row_filter_function", "macro_sql": "{%- macro quote_row_filter_function(function) -%}\n {#- Quote a fully qualified function name: catalog.schema.func -> `catalog`.`schema`.`func`\n Function names are stored raw internally; backticks are added at SQL generation time. -#}\n {%- set parts = function.split('.') -%}\n {%- if parts | length == 3 -%}\n `{{ parts[0] }}`.`{{ parts[1] }}`.`{{ parts[2] }}`\n {%- else -%}\n {#- Fallback for unexpected format -#}\n `{{ function }}`\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024173, "supported_languages": null}, "macro.dbt_databricks.alter_set_row_filter": {"name": "alter_set_row_filter", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.alter_set_row_filter", "macro_sql": "\n\n{%- macro alter_set_row_filter(relation, row_filter) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n SET ROW FILTER {{ quote_row_filter_function(row_filter.function) }}\n ON ({{ quote_row_filter_columns(row_filter.columns) }})\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.quote_row_filter_function", "macro.dbt_databricks.quote_row_filter_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024302, "supported_languages": null}, "macro.dbt_databricks.alter_drop_row_filter": {"name": "alter_drop_row_filter", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.alter_drop_row_filter", "macro_sql": "{%- macro alter_drop_row_filter(relation) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n DROP ROW FILTER\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024385, "supported_languages": null}, "macro.dbt_databricks.apply_row_filter": {"name": "apply_row_filter", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.apply_row_filter", "macro_sql": "\n\n{%- macro apply_row_filter(target_relation, row_filter) -%}\n {#- Executes SQL immediately via call statement('main') -#}\n {#- row_filter is a RowFilterConfig object with is_change, should_unset, function fields -#}\n {%- if target_relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Row filters are not supported for Hive Metastore\") }}\n {%- endif -%}\n\n {%- if row_filter.is_change -%}\n {%- if row_filter.should_unset -%}\n {%- call statement('main') -%}\n {{ alter_drop_row_filter(target_relation) }}\n {%- endcall -%}\n {%- elif row_filter.function -%}\n {%- call statement('main') -%}\n {{ alter_set_row_filter(target_relation, row_filter) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_drop_row_filter", "macro.dbt_databricks.alter_set_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024657, "supported_languages": null}, "macro.dbt_databricks.drop_row_filter_if_exists": {"name": "drop_row_filter_if_exists", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.drop_row_filter_if_exists", "macro_sql": "\n\n{%- macro drop_row_filter_if_exists(relation) -%}\n {#- Drops any existing row filter from a relation before CREATE OR REPLACE.\n Used by table materialization when the model has no row filter configured\n but the existing table might have one from a previous run. -#}\n {%- if not relation.is_hive_metastore() -%}\n {%- set existing_filters = fetch_row_filters(relation) -%}\n {%- if existing_filters | length > 0 -%}\n {%- call statement('drop_row_filter') -%}\n {{ alter_drop_row_filter(relation) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.fetch_row_filters", "macro.dbt.statement", "macro.dbt_databricks.alter_drop_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024835, "supported_languages": null}, "macro.dbt_databricks.qualify_row_filter_function": {"name": "qualify_row_filter_function", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.qualify_row_filter_function", "macro_sql": "\n\n{%- macro qualify_row_filter_function(function, relation) -%}\n {#- Handle 1-part and 3-part, reject 2-part. -#}\n {#- IMPORTANT: Keep in sync with Python _qualify_function_name() -#}\n {%- set parts = function.replace('`', '').split('.') -%}\n\n {%- if parts | length == 1 -%}\n {#- Unqualified: fn -> catalog.schema.fn -#}\n {{ relation.database }}.{{ relation.schema }}.{{ parts[0] }}\n {%- elif parts | length == 2 -%}\n {#- Ambiguous: reject -#}\n {{ exceptions.raise_compiler_error(\n \"Row filter function '\" ~ function ~ \"' is ambiguous. \" ~\n \"Use either unqualified name (e.g., 'my_filter') or \" ~\n \"fully qualified name (e.g., 'catalog.schema.my_filter').\"\n ) }}\n {%- elif parts | length == 3 -%}\n {#- Fully qualified -#}\n {{ parts[0] }}.{{ parts[1] }}.{{ parts[2] }}\n {%- else -%}\n {{ exceptions.raise_compiler_error(\n \"Row filter function '\" ~ function ~ \"' has too many parts.\"\n ) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.025177, "supported_languages": null}, "macro.dbt_databricks.get_create_row_filter_clause": {"name": "get_create_row_filter_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.get_create_row_filter_clause", "macro_sql": "{%- macro get_create_row_filter_clause(relation) -%}\n {%- set row_filter = config.get('row_filter') -%}\n\n {%- if not row_filter or not row_filter.get('function') -%}\n {#-- Model has no row filter - drop any existing filter from table --#}\n {{ drop_row_filter_if_exists(relation) }}\n {%- else -%}\n {#-- Model has row filter - generate the WITH ROW FILTER clause --#}\n {%- set columns = row_filter.get('columns', []) -%}\n\n {#- Normalize string to list -#}\n {%- if columns is string -%}\n {%- set columns = [columns] -%}\n {%- endif -%}\n\n {#- Validate columns is non-empty -#}\n {%- if not columns or columns | length == 0 -%}\n {{ exceptions.raise_compiler_error(\n \"Row filter function '\" ~ row_filter.get('function') ~ \"' requires a non-empty 'columns' value.\"\n ) }}\n {%- endif -%}\n\n {%- set function = qualify_row_filter_function(row_filter.get('function'), relation) -%}\n WITH ROW FILTER {{ quote_row_filter_function(function) }} ON ({{ quote_row_filter_columns(columns) }})\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.drop_row_filter_if_exists", "macro.dbt_databricks.qualify_row_filter_function", "macro.dbt_databricks.quote_row_filter_function", "macro.dbt_databricks.quote_row_filter_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.025575, "supported_languages": null}, "macro.dbt_databricks.row_filter_exists": {"name": "row_filter_exists", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.row_filter_exists", "macro_sql": "{%- macro row_filter_exists() -%}\n {%- set row_filter = config.get('row_filter') -%}\n {%- set has_function = row_filter and row_filter.get('function') -%}\n {%- set columns = row_filter.get('columns') if row_filter else none -%}\n\n {#- Normalize string to list -#}\n {%- if columns is string -%}\n {%- set columns = [columns] -%}\n {%- endif -%}\n\n {%- set has_columns = columns and columns | length > 0 -%}\n {%- do return(has_function and has_columns) -%}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.025833, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_partition_by": {"name": "get_create_sql_partition_by", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/partitioning.sql", "original_file_path": "macros/relations/components/partitioning.sql", "unique_id": "macro.dbt_databricks.get_create_sql_partition_by", "macro_sql": "{% macro get_create_sql_partition_by(partition_by) -%}\n{%- if partition_by -%}\n PARTITIONED BY ({%- for col in partition_by -%}{{ col }}{% if not loop.last %}, {% endif %}{%- endfor %})\n{%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0259998, "supported_languages": null}, "macro.dbt_databricks.fetch_column_tags": {"name": "fetch_column_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.fetch_column_tags", "macro_sql": "{% macro fetch_column_tags(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Column tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {% call statement('list_column_tags', fetch_result=True) -%}\n {{ fetch_column_tags_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_column_tags').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_column_tags_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.026499, "supported_languages": null}, "macro.dbt_databricks.fetch_column_tags_sql": {"name": "fetch_column_tags_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.fetch_column_tags_sql", "macro_sql": "{% macro fetch_column_tags_sql(relation) -%}\n SELECT \n column_name,\n tag_name,\n tag_value\n FROM `system`.`information_schema`.`column_tags`\n WHERE catalog_name = '{{ relation.database|lower }}'\n AND schema_name = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}';\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.026617, "supported_languages": null}, "macro.dbt_databricks.apply_column_tags": {"name": "apply_column_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.apply_column_tags", "macro_sql": "{% macro apply_column_tags(relation, column_tags) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Column tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {{ log(\"Applying column tags to relation \" ~ relation) }}\n {%- if column_tags.set_column_tags %}\n {%- for column, tags in column_tags.set_column_tags.items() -%}\n {%- call statement('main') -%}\n {{ alter_set_column_tags(relation, column, tags) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_set_column_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0269701, "supported_languages": null}, "macro.dbt_databricks.alter_set_column_tags": {"name": "alter_set_column_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.alter_set_column_tags", "macro_sql": "{% macro alter_set_column_tags(relation, column, tags) -%}\n {# ALTER VIEW does not support setting column tags, but ALTER TABLE works for views #}\n {%- if relation.type == 'view' -%}\n ALTER TABLE {{ relation.render() }}\n {%- else -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n {%- endif -%}\n ALTER COLUMN `{{ column }}`\n SET TAGS (\n {%- for tag_name, tag_value in tags.items() -%}\n '{{ tag_name }}' = '{{ tag_value }}'{%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.027207, "supported_languages": null}, "macro.dbt_databricks.column_tags_exist": {"name": "column_tags_exist", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.column_tags_exist", "macro_sql": "{% macro column_tags_exist() %}\n {% for column_name, column in model.columns.items() %}\n {% if column is mapping and column.get('databricks_tags') %}\n {{ return(true) }}\n {% endif %}\n {% endfor %}\n {{ return(false) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0273678, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_tblproperties": {"name": "get_create_sql_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/tblproperties.sql", "original_file_path": "macros/relations/components/tblproperties.sql", "unique_id": "macro.dbt_databricks.get_create_sql_tblproperties", "macro_sql": "{% macro get_create_sql_tblproperties(tblproperties) %}\n {{ databricks__tblproperties_clause(tblproperties)}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.027458, "supported_languages": null}, "macro.dbt_databricks.fetch_column_masks": {"name": "fetch_column_masks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.fetch_column_masks", "macro_sql": "{% macro fetch_column_masks(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Column masks are not supported for Hive Metastore\") }}\n {%- endif %}\n {% call statement('list_column_masks', fetch_result=True) -%}\n {{ fetch_column_masks_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_column_masks').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_column_masks_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.027842, "supported_languages": null}, "macro.dbt_databricks.fetch_column_masks_sql": {"name": "fetch_column_masks_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.fetch_column_masks_sql", "macro_sql": "{% macro fetch_column_masks_sql(relation) -%}\n SELECT \n column_name,\n mask_name,\n using_columns\n FROM `system`.`information_schema`.`column_masks`\n WHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}';\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.02794, "supported_languages": null}, "macro.dbt_databricks.apply_column_masks": {"name": "apply_column_masks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.apply_column_masks", "macro_sql": "{% macro apply_column_masks(relation, column_masks) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Column masks are not supported for Hive Metastore\") }}\n {%- endif %}\n {{ log(\"Applying column masks to relation \" ~ relation) }}\n {%- if column_masks.unset_column_masks %}\n {%- for column in column_masks.unset_column_masks -%}\n {%- call statement('main') -%}\n {{ alter_drop_column_mask(relation, column) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n {%- if column_masks.set_column_masks %}\n {%- for column, mask in column_masks.set_column_masks.items() -%}\n {%- call statement('main') -%}\n {{ alter_set_column_mask(relation, column, mask) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_drop_column_mask", "macro.dbt_databricks.alter_set_column_mask"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.02824, "supported_languages": null}, "macro.dbt_databricks.alter_drop_column_mask": {"name": "alter_drop_column_mask", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.alter_drop_column_mask", "macro_sql": "{% macro alter_drop_column_mask(relation, column) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n ALTER COLUMN `{{ column }}`\n DROP MASK;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0283341, "supported_languages": null}, "macro.dbt_databricks.alter_set_column_mask": {"name": "alter_set_column_mask", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.alter_set_column_mask", "macro_sql": "{% macro alter_set_column_mask(relation, column, mask) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n ALTER COLUMN `{{ column }}`\n SET MASK {{ mask.function }}\n {%- if mask.using_columns %}\n USING COLUMNS ({{ mask.using_columns }})\n {%- endif %};\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0284858, "supported_languages": null}, "macro.dbt_databricks.column_mask_exists": {"name": "column_mask_exists", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.column_mask_exists", "macro_sql": "{% macro column_mask_exists() %}\n {% for column_name, column in model.columns.items() %}\n {% if column is mapping and column.get('column_mask') %}\n {{ return(true) }}\n {% endif %}\n {% endfor %}\n {{ return(false) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.02866, "supported_languages": null}, "macro.dbt_databricks.fetch_non_null_constraint_columns": {"name": "fetch_non_null_constraint_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_non_null_constraint_columns", "macro_sql": "{% macro fetch_non_null_constraint_columns(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Incremental application of constraints is not supported for Hive Metastore\") }}\n {%- endif %}\n {% call statement('list_non_null_constraint_columns', fetch_result=True) -%}\n {{ fetch_non_null_constraint_columns_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_non_null_constraint_columns').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_non_null_constraint_columns_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.029232, "supported_languages": null}, "macro.dbt_databricks.fetch_non_null_constraint_columns_sql": {"name": "fetch_non_null_constraint_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_non_null_constraint_columns_sql", "macro_sql": "{% macro fetch_non_null_constraint_columns_sql(relation) -%}\n SELECT column_name\n FROM `{{ relation.database|lower }}`.`information_schema`.`columns`\n WHERE table_catalog = '{{ relation.database|lower }}' \n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n AND is_nullable = 'NO';\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0293398, "supported_languages": null}, "macro.dbt_databricks.fetch_primary_key_constraints": {"name": "fetch_primary_key_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_primary_key_constraints", "macro_sql": "{% macro fetch_primary_key_constraints(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Incremental application of constraints is not supported for Hive Metastore\") }}\n {%- endif %}\n {% call statement('list_primary_key_constraints', fetch_result=True) -%}\n {{ fetch_primary_key_constraints_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_primary_key_constraints').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_primary_key_constraints_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.029509, "supported_languages": null}, "macro.dbt_databricks.fetch_primary_key_constraints_sql": {"name": "fetch_primary_key_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_primary_key_constraints_sql", "macro_sql": "{% macro fetch_primary_key_constraints_sql(relation) -%}\n SELECT kcu.constraint_name, kcu.column_name\n FROM `{{ relation.database|lower }}`.information_schema.key_column_usage kcu\n WHERE kcu.table_catalog = '{{ relation.database|lower }}' \n AND kcu.table_schema = '{{ relation.schema|lower }}'\n AND kcu.table_name = '{{ relation.identifier|lower }}' \n AND kcu.constraint_name = (\n SELECT constraint_name\n FROM `{{ relation.database|lower }}`.information_schema.table_constraints\n WHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}' \n AND constraint_type = 'PRIMARY KEY'\n )\n ORDER BY kcu.ordinal_position;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0296829, "supported_languages": null}, "macro.dbt_databricks.fetch_foreign_key_constraints": {"name": "fetch_foreign_key_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_foreign_key_constraints", "macro_sql": "{% macro fetch_foreign_key_constraints(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Incremental application of constraints is not supported for Hive Metastore\") }}\n {%- endif %}\n {% call statement('list_foreign_key_constraints', fetch_result=True) -%}\n {{ fetch_foreign_key_constraints_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_foreign_key_constraints').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_foreign_key_constraints_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.029862, "supported_languages": null}, "macro.dbt_databricks.fetch_foreign_key_constraints_sql": {"name": "fetch_foreign_key_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_foreign_key_constraints_sql", "macro_sql": "{% macro fetch_foreign_key_constraints_sql(relation) -%}\n SELECT\n kcu.constraint_name,\n kcu.column_name AS from_column,\n ukcu.table_catalog AS to_catalog,\n ukcu.table_schema AS to_schema,\n ukcu.table_name AS to_table,\n ukcu.column_name AS to_column\n FROM `{{ relation.database|lower }}`.information_schema.key_column_usage kcu\n JOIN `{{ relation.database|lower }}`.information_schema.referential_constraints rc\n ON kcu.constraint_name = rc.constraint_name\n JOIN `{{ relation.database|lower }}`.information_schema.key_column_usage ukcu\n ON rc.unique_constraint_name = ukcu.constraint_name\n AND kcu.ordinal_position = ukcu.ordinal_position\n WHERE kcu.table_catalog = '{{ relation.database|lower }}'\n AND kcu.table_schema = '{{ relation.schema|lower }}'\n AND kcu.table_name = '{{ relation.identifier|lower }}'\n AND kcu.constraint_name IN (\n SELECT constraint_name\n FROM `{{ relation.database|lower }}`.information_schema.table_constraints\n WHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n AND constraint_type = 'FOREIGN KEY'\n )\n ORDER BY kcu.ordinal_position;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.030079, "supported_languages": null}, "macro.dbt_databricks.apply_constraints": {"name": "apply_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.apply_constraints", "macro_sql": "{% macro apply_constraints(relation, constraints) -%}\n {{ log(\"Applying constraints to relation \" ~ constraints) }}\n {%- if constraints and relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Constraints are only supported for Unity Catalog\") }}\n {%- endif -%}\n {# Order matters here because key constraints depend on non-null constraints #} \n {%- if constraints.unset_constraints %}\n {%- for constraint in constraints.unset_constraints -%}\n {%- call statement('main') -%}\n {{ alter_unset_constraint(relation, constraint) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n {%- if constraints.unset_non_nulls %}\n {%- for column in constraints.unset_non_nulls -%}\n {%- call statement('main') -%}\n {{ alter_unset_non_null_constraint(relation, column) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n {%- if constraints.set_non_nulls %}\n {%- for column in constraints.set_non_nulls -%}\n {%- call statement('main') -%}\n {{ alter_set_non_null_constraint(relation, column) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n {%- if constraints.set_constraints %}\n {%- for constraint in constraints.set_constraints -%}\n {%- call statement('main') -%}\n {{ alter_set_constraint(relation, constraint) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_unset_constraint", "macro.dbt_databricks.alter_unset_non_null_constraint", "macro.dbt_databricks.alter_set_non_null_constraint", "macro.dbt_databricks.alter_set_constraint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.03053, "supported_languages": null}, "macro.dbt_databricks.alter_set_non_null_constraint": {"name": "alter_set_non_null_constraint", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.alter_set_non_null_constraint", "macro_sql": "{% macro alter_set_non_null_constraint(relation, column) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }} ALTER COLUMN {{ adapter.quote(column) }} SET NOT NULL;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0306408, "supported_languages": null}, "macro.dbt_databricks.alter_unset_non_null_constraint": {"name": "alter_unset_non_null_constraint", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.alter_unset_non_null_constraint", "macro_sql": "{% macro alter_unset_non_null_constraint(relation, column) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }} ALTER COLUMN {{ adapter.quote(column) }} DROP NOT NULL;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0307462, "supported_languages": null}, "macro.dbt_databricks.alter_set_constraint": {"name": "alter_set_constraint", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.alter_set_constraint", "macro_sql": "{% macro alter_set_constraint(relation, constraint) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }} ADD {{ constraint.render() }};\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.030848, "supported_languages": null}, "macro.dbt_databricks.alter_unset_constraint": {"name": "alter_unset_constraint", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.alter_unset_constraint", "macro_sql": "{% macro alter_unset_constraint(relation, constraint) -%}\n {% set constraint_type = constraint.type %}\n {% if constraint_type == 'primary_key' %}\n {# Need to only add CASCADE to PK constraints because dropping check constraints break when adding CASCADE #}\n ALTER {{ relation.type.render() }} {{ relation.render() }} DROP CONSTRAINT {{ constraint.name }} CASCADE;\n {% else %}\n ALTER {{ relation.type.render() }} {{ relation.render() }} DROP CONSTRAINT IF EXISTS {{ constraint.name }};\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.031043, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_table": {"name": "databricks__drop_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_table", "macro_sql": "{% macro databricks__drop_table(relation) -%}\n drop table if exists {{ relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.031122, "supported_languages": null}, "macro.dbt_databricks.safe_relation_replace": {"name": "safe_relation_replace", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt_databricks.safe_relation_replace", "macro_sql": "{% macro safe_relation_replace(existing_relation, staging_relation, intermediate_relation, compiled_code) %}\n \n {{ create_table_at(staging_relation, intermediate_relation, compiled_code) }}\n\n {{ create_backup(existing_relation) }}\n\n {{ adapter.rename_relation(staging_relation, existing_relation) }}\n\n \n\n {% call statement('main') %}\n {{ get_drop_backup_sql(existing_relation) }}\n {% endcall %}\n \n {{ adapter.cache_dropped(make_backup_relation(existing_relation, existing_relation.type)) }}\n\n {{ drop_relation_if_exists(intermediate_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_table_at", "macro.dbt_databricks.create_backup", "macro.dbt.statement", "macro.dbt.get_drop_backup_sql", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.031401, "supported_languages": null}, "macro.dbt_databricks.databricks__get_rename_table_sql": {"name": "databricks__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt_databricks.databricks__get_rename_table_sql", "macro_sql": "{% macro databricks__get_rename_table_sql(relation, new_name) %}\n ALTER TABLE {{ relation.render() }} RENAME TO `{{ new_name }}`\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.031498, "supported_languages": null}, "macro.dbt_databricks.apply_config_changeset": {"name": "apply_config_changeset", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/alter.sql", "original_file_path": "macros/relations/table/alter.sql", "unique_id": "macro.dbt_databricks.apply_config_changeset", "macro_sql": "{% macro apply_config_changeset(target_relation, model, configuration_changes) %}\n {{ log(\"Applying configuration changes to relation \" ~ target_relation) }}\n {% if configuration_changes %}\n {% set comment = configuration_changes.changes.get(\"comment\") %}\n {% set column_comments = configuration_changes.changes.get(\"column_comments\") %}\n {% set column_tags = configuration_changes.changes.get(\"column_tags\") %}\n {% set tags = configuration_changes.changes.get(\"tags\") %}\n {% set tblproperties = configuration_changes.changes.get(\"tblproperties\") %}\n {% set liquid_clustering = configuration_changes.changes.get(\"liquid_clustering\")%}\n {% set constraints = configuration_changes.changes.get(\"constraints\") %}\n {% set column_masks = configuration_changes.changes.get(\"column_masks\") %}\n {% set row_filter = configuration_changes.changes.get(\"row_filter\") %}\n {% if tags is not none %}\n {% do apply_tags(target_relation, tags.set_tags) %}\n {%- endif -%}\n {% if tblproperties is not none %}\n {% do apply_tblproperties(target_relation, tblproperties.tblproperties) %}\n {%- endif -%}\n {% if liquid_clustering is not none %}\n {% do apply_liquid_clustered_cols(target_relation, liquid_clustering) %}\n {%- endif -%}\n {% if comment %}\n {{ run_query_as(alter_relation_comment_sql(target_relation, comment.comment), 'alter_relation_comment', fetch_result=False) }}\n {% endif %}\n {% if column_comments %}\n {{ alter_column_comments(target_relation, column_comments.comments) }}\n {% endif %}\n {% if column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n {% if constraints %}\n {{ apply_constraints(target_relation, constraints) }}\n {% endif %}\n {% if column_masks %}\n {{ apply_column_masks(target_relation, column_masks) }}\n {% endif %}\n {% if row_filter %}\n {{ apply_row_filter(target_relation, row_filter) }}\n {% endif %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.apply_liquid_clustered_cols", "macro.dbt_databricks.run_query_as", "macro.dbt_databricks.alter_relation_comment_sql", "macro.dbt_databricks.alter_column_comments", "macro.dbt_databricks.apply_column_tags", "macro.dbt_databricks.apply_constraints", "macro.dbt_databricks.apply_column_masks", "macro.dbt_databricks.apply_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.032541, "supported_languages": null}, "macro.dbt_databricks.create_table_at": {"name": "create_table_at", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.create_table_at", "macro_sql": "{% macro create_table_at(relation, intermediate_relation, compiled_code) %}\n {% set tags = config.get('databricks_tags') %}\n {% set model_columns = model.get('columns', []) %}\n {% set existing_columns = adapter.get_columns_in_relation(intermediate_relation) %}\n {% set contract_config = config.get('contract') %}\n {% set contract_enforced = contract_config and contract_config.enforced %}\n {% if contract_enforced %}\n {% set model_constraints = model.get('constraints', []) %}\n {% else %}\n {% set model_constraints = [] %}\n {% endif %}\n {% set columns_and_constraints = adapter.parse_columns_and_constraints(existing_columns, model_columns, model_constraints, contract_enforced, model.name) %}\n {% set target_relation = relation.enrich(columns_and_constraints[1]) %}\n \n {% call statement('main') %}\n {{ get_create_table_sql(target_relation, columns_and_constraints[0], compiled_code) }}\n {% endcall %}\n\n {{ apply_alter_constraints(target_relation) }}\n {{ apply_tags(target_relation, tags) }}\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {% call statement('merge into target') %}\n insert into {{ target_relation }} by name select * from {{ intermediate_relation }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_create_table_sql", "macro.dbt_databricks.apply_alter_constraints", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0335948, "supported_languages": null}, "macro.dbt_databricks.get_create_table_sql": {"name": "get_create_table_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.get_create_table_sql", "macro_sql": "{% macro get_create_table_sql(target_relation, columns, compiled_code) %}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n {%- set contract = config.get('contract') -%}\n {%- set contract_enforced = contract and contract.enforced -%}\n {%- if contract_enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- endif -%}\n\n {%- if catalog_relation.file_format in ('delta', 'iceberg') %}\n create or replace table {{ target_relation.render() }}\n {% else %}\n create table {{ target_relation.render() }}\n {% endif -%}\n {{ get_column_and_constraints_sql(target_relation, columns) }}\n {{ file_format_clause(catalog_relation) }}\n {{ databricks__options_clause(catalog_relation) }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ get_create_row_filter_clause(target_relation) }}\n {{ liquid_clustered_cols() }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(catalog_relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt_databricks.get_column_and_constraints_sql", "macro.dbt_databricks.file_format_clause", "macro.dbt_databricks.databricks__options_clause", "macro.dbt_spark.partition_cols", "macro.dbt_databricks.get_create_row_filter_clause", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.034013, "supported_languages": null}, "macro.dbt_databricks.databricks__create_table_as": {"name": "databricks__create_table_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_table_as", "macro_sql": "{% macro databricks__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if catalog_relation.file_format == 'delta' %}\n create or replace table {{ relation.render() }}\n {% else %}\n create table {{ relation.render() }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause(catalog_relation) }}\n {{ databricks__options_clause(catalog_relation) }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ get_create_row_filter_clause(relation) }}\n {{ liquid_clustered_cols() }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(catalog_relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ databricks__py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_databricks.file_format_clause", "macro.dbt_databricks.databricks__options_clause", "macro.dbt_spark.partition_cols", "macro.dbt_databricks.get_create_row_filter_clause", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt_databricks.databricks__py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.034562, "supported_languages": null}, "macro.dbt_databricks.databricks__options_clause": {"name": "databricks__options_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__options_clause", "macro_sql": "{% macro databricks__options_clause(catalog_relation=none) -%}\n {#-\n Moving forward, this macro should require a `catalog_relation`, which is covered by the first condition.\n However, there could be existing macros that is still passing no arguments, including user macros.\n Hence, we need to support the old code still, which is covered by the second condition.\n Additionally, since this rolls up to `options_clause` in `dbt-spark`, which does not have any arguments,\n all calls to `options_clause` will take the second path. This macro needs to be called directly\n via `databricks__options_clause`.\n -#}\n {%- if catalog_relation is not none -%}\n {%- set file_format = catalog_relation.file_format -%}\n {%- else -%}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n {%- endif -%}\n\n {%- set options = config.get('options') -%}\n {%- if file_format == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035139, "supported_languages": null}, "macro.dbt_databricks.get_create_intermediate_table": {"name": "get_create_intermediate_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.get_create_intermediate_table", "macro_sql": "{% macro get_create_intermediate_table(relation, compiled_code, language) %}\n {%- if language == 'sql' -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {{ create_python_intermediate_table(relation, compiled_code) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt_databricks.create_python_intermediate_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035273, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_view": {"name": "databricks__drop_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_view", "macro_sql": "{% macro databricks__drop_view(relation) -%}\n DROP VIEW IF EXISTS {{ relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035372, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_view_sql": {"name": "databricks__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_view_sql", "macro_sql": "{% macro databricks__get_replace_view_sql(target_relation, sql) %}\n {{ create_view_as(target_relation, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035466, "supported_languages": null}, "macro.dbt_databricks.databricks__get_rename_view_sql": {"name": "databricks__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt_databricks.databricks__get_rename_view_sql", "macro_sql": "{% macro databricks__get_rename_view_sql(relation, new_name) %}\n ALTER VIEW {{ relation.render() }} RENAME TO {{ new_name }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035554, "supported_languages": null}, "macro.dbt_databricks.alter_view": {"name": "alter_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/alter.sql", "original_file_path": "macros/relations/view/alter.sql", "unique_id": "macro.dbt_databricks.alter_view", "macro_sql": "{% macro alter_view(target_relation, changes) %}\n {{ log(\"Updating view via ALTER\") }}\n {{ adapter.dispatch('alter_view', 'dbt')(target_relation, changes) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035757, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_view": {"name": "databricks__alter_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/alter.sql", "original_file_path": "macros/relations/view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__alter_view", "macro_sql": "{% macro databricks__alter_view(target_relation, changes) %}\n {% set tags = changes.get(\"tags\") %}\n {% set tblproperties = changes.get(\"tblproperties\") %}\n {% set query = changes.get(\"query\") %}\n {% set column_comments = changes.get(\"column_comments\") %}\n {% if tags %}\n {{ apply_tags(target_relation, tags.set_tags) }}\n {% endif %}\n {% if tblproperties %}\n {{ apply_tblproperties(target_relation, tblproperties.tblproperties) }}\n {% endif %}\n {% if query %}\n {{ alter_query(target_relation, query.query) }}\n {% endif %}\n {% if column_comments %}\n {{ alter_column_comments(target_relation, column_comments.comments) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.alter_query", "macro.dbt_databricks.alter_column_comments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.036093, "supported_languages": null}, "macro.dbt_databricks.databricks__create_view_as": {"name": "databricks__create_view_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_view_as", "macro_sql": "{% macro databricks__create_view_as(relation, sql) %}\n {% if column_mask_exists() %}\n {% do exceptions.raise_compiler_error(\"Column masks are not supported for views.\") %}\n {% endif %}\n {% if row_filter_exists() %}\n {% do exceptions.raise_compiler_error(\"Row filters are not supported for views.\") %}\n {% endif %}\n {{ log(\"Creating view \" ~ relation) }}\n create or replace view {{ relation.render() }}\n {%- if config.persist_column_docs() -%}\n {%- set model_columns = model.columns -%}\n {%- set query_columns = get_columns_in_query(sql) -%}\n {%- if query_columns %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {%- endif -%}\n {%- endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif -%}\n {{ tblproperties_clause() }}\n as (\n {{ sql }}\n )\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.column_mask_exists", "macro.dbt_databricks.row_filter_exists", "macro.dbt.get_columns_in_query", "macro.dbt_databricks.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent", "macro.dbt_databricks.tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.037025, "supported_languages": null}, "macro.dbt_databricks.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {%- if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {%- set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") -%}\n {%- set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n {%- else -%}\n {{ adapter.quote(column_name) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.037243, "supported_languages": null}, "macro.dbt_databricks.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) -%}\n {%- for column_name in query_columns -%}\n {{ get_column_comment_sql(column_name, model_columns) }}{{\",\\n\\t\" if not loop.last else \"\" }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.037353, "supported_languages": null}, "macro.dbt_databricks.get_replace_metric_view_sql": {"name": "get_replace_metric_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/replace.sql", "original_file_path": "macros/relations/metric_view/replace.sql", "unique_id": "macro.dbt_databricks.get_replace_metric_view_sql", "macro_sql": "{% macro get_replace_metric_view_sql(target_relation, sql) %}\n {{ adapter.dispatch('get_replace_metric_view_sql', 'dbt')(target_relation, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_metric_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0374742, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_metric_view_sql": {"name": "databricks__get_replace_metric_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/replace.sql", "original_file_path": "macros/relations/metric_view/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_metric_view_sql", "macro_sql": "{% macro databricks__get_replace_metric_view_sql(target_relation, sql) %}\n {{ get_create_metric_view_as_sql(target_relation, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_metric_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0375402, "supported_languages": null}, "macro.dbt_databricks.alter_metric_view": {"name": "alter_metric_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.alter_metric_view", "macro_sql": "{% macro alter_metric_view(target_relation, changes) %}\n {{ log(\"Updating metric view via ALTER\") }}\n {{ adapter.dispatch('alter_metric_view', 'dbt')(target_relation, changes) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_metric_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.037801, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_metric_view": {"name": "databricks__alter_metric_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__alter_metric_view", "macro_sql": "{% macro databricks__alter_metric_view(target_relation, changes) %}\n {% set tags = changes.get(\"tags\") %}\n {% set tblproperties = changes.get(\"tblproperties\") %}\n {% set query = changes.get(\"query\") %}\n\n {# Handle YAML definition changes via ALTER VIEW AS #}\n {% if query %}\n {% call statement('main') %}\n {{ get_alter_metric_view_as_sql(target_relation, query.query) }}\n {% endcall %}\n {% else %}\n {# Ensure statement('main') is called for dbt to track the run #}\n {% call statement('main') %}\n select 1\n {% endcall %}\n {% endif %}\n\n {% if tags %}\n {{ apply_tags(target_relation, tags.set_tags) }}\n {% endif %}\n {% if tblproperties %}\n {{ apply_tblproperties(target_relation, tblproperties.tblproperties) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_alter_metric_view_as_sql", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038118, "supported_languages": null}, "macro.dbt_databricks.get_alter_metric_view_as_sql": {"name": "get_alter_metric_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_metric_view_as_sql", "macro_sql": "{% macro get_alter_metric_view_as_sql(relation, yaml_content) -%}\n {{ adapter.dispatch('get_alter_metric_view_as_sql', 'dbt')(relation, yaml_content) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_metric_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038201, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_metric_view_as_sql": {"name": "databricks__get_alter_metric_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_metric_view_as_sql", "macro_sql": "{% macro databricks__get_alter_metric_view_as_sql(relation, yaml_content) %}\n{%- set yaml_body = adapter.yaml_quote_backtick_values(yaml_content) -%}\nalter view {{ relation.render() }} as $$\n{{ yaml_body }}\n$$\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038293, "supported_languages": null}, "macro.dbt_databricks.replace_with_metric_view": {"name": "replace_with_metric_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.replace_with_metric_view", "macro_sql": "{% macro replace_with_metric_view(existing_relation, target_relation) %}\n {% set sql = adapter.clean_sql(sql) %}\n {% set tags = config.get('databricks_tags') %}\n {% set tblproperties = config.get('tblproperties') %}\n {{ execute_multiple_statements(get_replace_sql(existing_relation, target_relation, sql)) }}\n {%- do apply_tags(target_relation, tags) -%}\n\n {% if tblproperties %}\n {{ apply_tblproperties(target_relation, tblproperties) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.execute_multiple_statements", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038506, "supported_languages": null}, "macro.dbt_databricks.get_create_metric_view_as_sql": {"name": "get_create_metric_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/create.sql", "original_file_path": "macros/relations/metric_view/create.sql", "unique_id": "macro.dbt_databricks.get_create_metric_view_as_sql", "macro_sql": "{% macro get_create_metric_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_metric_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_metric_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038629, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_metric_view_as_sql": {"name": "databricks__get_create_metric_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/create.sql", "original_file_path": "macros/relations/metric_view/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_metric_view_as_sql", "macro_sql": "{% macro databricks__get_create_metric_view_as_sql(relation, sql) %}\n{%- set yaml_body = adapter.yaml_quote_backtick_values(sql) -%}\ncreate or replace view {{ relation.render() }}\nwith metrics\nlanguage yaml\nas $$\n{{ yaml_body }}\n$$\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0387242, "supported_languages": null}, "macro.dbt_databricks.statement_with_staging_table": {"name": "statement_with_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.statement_with_staging_table", "macro_sql": "{% macro statement_with_staging_table(name=None, staging_table=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result, staging_table=staging_table) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039202, "supported_languages": null}, "macro.dbt_databricks.execute_multiple_statements": {"name": "execute_multiple_statements", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.execute_multiple_statements", "macro_sql": "{% macro execute_multiple_statements(statements) %}\n {%- if statements is string %}\n {% call statement(name=\"main\") %}\n {{ statements }}\n {% endcall %}\n {%- else %}\n {%- for sql in statements %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039372, "supported_languages": null}, "macro.dbt_databricks.run_query_as": {"name": "run_query_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.run_query_as", "macro_sql": "{% macro run_query_as(sql, name, fetch_result=True) %}\n {% call statement(name, fetch_result, auto_begin=False) %}\n {{ sql }}\n {% endcall %}\n\n {% if fetch_result %}\n {{ return(load_result(name).table) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039519, "supported_languages": null}, "macro.dbt_databricks.databricks__dateadd": {"name": "databricks__dateadd", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_databricks.databricks__dateadd", "macro_sql": "{% macro databricks__dateadd(datepart, interval, from_date_or_timestamp) %}\n {%- if adapter.has_dbr_capability('timestampdiff') -%}\n timestampadd({{datepart}}, {{interval}}, {{from_date_or_timestamp}})\n {%- else -%}\n {{ spark__dateadd(datepart, interval, from_date_or_timestamp) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039696, "supported_languages": null}, "macro.dbt_databricks.databricks__datediff": {"name": "databricks__datediff", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_databricks.databricks__datediff", "macro_sql": "{% macro databricks__datediff(first_date, second_date, datepart) %}\n {%- if adapter.has_dbr_capability('timestampdiff') -%}\n timestampdiff({{datepart}}, {{date_trunc(datepart, first_date)}}, {{date_trunc(datepart, second_date)}})\n {%- else -%}\n {{ spark__datediff(first_date, second_date, datepart) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.date_trunc", "macro.dbt_spark.spark__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039925, "supported_languages": null}, "macro.dbt_databricks.databricks__split_part": {"name": "databricks__split_part", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_databricks.databricks__split_part", "macro_sql": "{% macro databricks__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% if part_number >= 0 %}\n\n {% set split_part_expr %}\n\n get(split(\n {{ string_text }},\n {{ delimiter_expr }}\n ), {{ part_number - 1 if part_number > 0 else part_number }})\n\n {% endset %}\n\n {% else %}\n\n {% set split_part_expr %}\n\n get(split(\n {{ string_text }},\n {{ delimiter_expr }}\n ), \n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1 + {{ part_number }}\n )\n\n {% endset %}\n\n {% endif %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0403001, "supported_languages": null}, "macro.dbt_databricks.databricks_copy_into": {"name": "databricks_copy_into", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/copy_into.sql", "original_file_path": "macros/adapters/copy_into.sql", "unique_id": "macro.dbt_databricks.databricks_copy_into", "macro_sql": "{% macro databricks_copy_into(\n target_table,\n source,\n file_format,\n expression_list=none,\n source_credential=none,\n source_encryption=none,\n validate=none,\n files=none,\n pattern=none,\n format_options=none,\n copy_options=none) -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=target.database,\n schema=target.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- set source_clause -%}\n {%- if expression_list -%}\n ( select {{ expression_list }} from '{{ source }}' )\n {%- else -%}\n '{{ source }}'\n {%- endif -%}\n {%- if source_credential or source_encryption %}\n WITH (\n {%- if source_credential %}\n credential (\n {%- for name in source_credential -%}\n '{{ name }}' = '{{ source_credential[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {%- if source_encryption %}\n encryption (\n {%- for name in source_encryption -%}\n '{{ name }}' = '{{ source_encryption[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n )\n {%- endif -%}\n {%- endset -%}\n\n {% set query %}\n copy into {{ target_relation }}\n from {{ source_clause }}\n fileformat = {{ file_format }}\n {% if validate -%} validate {{ validate }} {%- endif %}\n {% if files and pattern %}\n {{ exceptions.raise_compiler_error(\"You can only specify one of 'files' or 'pattern'\") }}\n {% endif %}\n {% if files -%}\n files = (\n {%- for file in files -%}\n '{{ file }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if pattern -%}\n pattern = '{{ pattern }}'\n {%- endif %}\n {% if format_options -%}\n format_options (\n {%- for key in format_options -%}\n '{{ key }}' = '{{ format_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if copy_options -%}\n copy_options (\n {%- for key in copy_options -%}\n '{{ key }}' = '{{ copy_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% endset %}\n\n {{ run_query_as(query, 'copy_into', fetch_result=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt_databricks.run_query_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.041465, "supported_languages": null}, "macro.dbt_databricks.make_staging_relation": {"name": "make_staging_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.make_staging_relation", "macro_sql": "{% macro make_staging_relation(base_relation, suffix='__dbt_stg', type='table') %}\n {% set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool %}\n {% if unique_tmp_table_suffix %}\n {% set suffix = adapter.generate_unique_temporary_table_suffix(suffix) %}\n {% endif %}\n {% set stg_identifier = base_relation.identifier ~ suffix %}\n {% set stg_relation = api.Relation.create(database=base_relation.database, schema=base_relation.schema, identifier=stg_identifier, type=type) %}\n {% do return(stg_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.041954, "supported_languages": null}, "macro.dbt_databricks.databricks__make_intermediate_relation": {"name": "databricks__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__make_intermediate_relation", "macro_sql": "{% macro databricks__make_intermediate_relation(base_relation, suffix) %}\n {{ return(databricks__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.042033, "supported_languages": null}, "macro.dbt_databricks.databricks__make_temp_relation": {"name": "databricks__make_temp_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__make_temp_relation", "macro_sql": "{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {% set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool %}\n\n {% if unique_tmp_table_suffix %}\n {% set suffix = adapter.generate_unique_temporary_table_suffix() %}\n {% endif %}\n \n {% if suffix == '__dbt_tmp' and model.batch %}\n {% set suffix = suffix ~ '_' ~ model.batch.id %}\n {% endif %}\n\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set language = model['language'] %}\n {%- if language == 'sql' -%}\n {% set temporary = not base_relation.is_hive_metastore() %}\n {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view', temporary=temporary) %}\n {%- else -%}\n {% set tmp_relation = api.Relation.create(database=base_relation.database, schema=base_relation.schema, identifier=tmp_identifier, type='table') %}\n {%- endif -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.042485, "supported_languages": null}, "macro.dbt_databricks.databricks__get_or_create_relation": {"name": "databricks__get_or_create_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__get_or_create_relation", "macro_sql": "{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}\n {%- set target_relation = adapter.get_relation(\n database=database,\n schema=schema,\n identifier=identifier,\n needs_information=needs_information) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type,\n temporary=False\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04276, "supported_languages": null}, "macro.dbt_databricks.get_column_and_constraints_sql": {"name": "get_column_and_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.get_column_and_constraints_sql", "macro_sql": "{% macro get_column_and_constraints_sql(relation, columns) %}\n (\n {% for column in columns %}\n {{ column.render_for_create() }}{% if not loop.last or relation.create_constraints %},{% endif %}\n {% endfor %}\n {% if relation.create_constraints %}\n {{ relation.render_constraints_for_create() }}\n {% endif %}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.042932, "supported_languages": null}, "macro.dbt_databricks.load_relation_with_metadata": {"name": "load_relation_with_metadata", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.load_relation_with_metadata", "macro_sql": "{% macro load_relation_with_metadata(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier,\n needs_information=True\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0430431, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog": {"name": "databricks__get_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog", "macro_sql": "{% macro databricks__get_catalog(information_schema, schemas) -%}\n {% set query %}\nWITH tables AS (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n),\ncolumns AS (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n)\n{{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query_as(query, 'get_catalog')) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt_databricks.run_query_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043443, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations": {"name": "databricks__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations", "macro_sql": "{% macro databricks__get_catalog_relations(information_schema, relations) -%}\n {% set query %}\nWITH tables AS (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(information_schema.database, relations) }}\n),\ncolumns AS (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(information_schema.database, relations) }}\n)\n{{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query_as(query, 'get_catalog_relations')) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt_databricks.run_query_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043628, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_tables_sql": {"name": "databricks__get_catalog_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro_sql": "{% macro databricks__get_catalog_tables_sql(information_schema) -%}\nSELECT\n table_catalog AS table_database,\n table_schema,\n table_name,\n lower(table_type) AS table_type,\n comment AS table_comment,\n table_owner,\n 'Last Modified' AS `stats:last_modified:label`,\n last_altered AS `stats:last_modified:value`,\n 'The timestamp for last update/change' AS `stats:last_modified:description`,\n (last_altered IS NOT NULL AND table_type NOT ILIKE '%VIEW%') AS `stats:last_modified:include`\nFROM `system`.`information_schema`.`tables`\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0436962, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_columns_sql": {"name": "databricks__get_catalog_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro_sql": "{% macro databricks__get_catalog_columns_sql(information_schema) -%}\nSELECT\n table_catalog AS table_database,\n table_schema,\n table_name,\n column_name,\n ordinal_position AS column_index,\n lower(full_data_type) AS column_type,\n comment AS column_comment\nFROM `system`.`information_schema`.`columns`\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043746, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_results_sql": {"name": "databricks__get_catalog_results_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro_sql": "{% macro databricks__get_catalog_results_sql() -%}\nSELECT *\nFROM tables\nJOIN columns USING (table_database, table_schema, table_name)\nORDER BY column_index\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043786, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql": {"name": "databricks__get_catalog_schemas_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_schemas_where_clause_sql(catalog, schemas) -%}\nWHERE table_catalog = '{{ catalog|lower }}' AND (\n {%- for relation in schemas -%}\n table_schema = '{{ relation[1]|lower }}'{%- if not loop.last %} OR {% endif -%}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043921, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql": {"name": "databricks__get_catalog_relations_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_relations_where_clause_sql(catalog, relations) -%}\nWHERE table_catalog = '{{ catalog|lower }}' AND (\n {%- for relation in relations -%}\n {%- if relation.schema and relation.identifier %}\n (\n table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n )\n {%- elif relation.schema %}\n (\n table_schema = '{{ relation.schema|lower }}'\n )\n {% else %}\n {% do exceptions.raise_compiler_error(\n '`get_catalog_relations` requires a list of relations, each with a schema'\n ) %}\n {% endif %}\n {%- if not loop.last %} OR {% endif -%}\n {%- endfor -%}\n)\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044159, "supported_languages": null}, "macro.dbt_databricks.current_catalog": {"name": "current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.current_catalog", "macro_sql": "{% macro current_catalog() -%}\n {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__current_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044322, "supported_languages": null}, "macro.dbt_databricks.databricks__current_catalog": {"name": "databricks__current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__current_catalog", "macro_sql": "{% macro databricks__current_catalog() -%}\n {{ return(run_query_as(current_catalog_sql(), 'current_catalog')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.current_catalog_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044396, "supported_languages": null}, "macro.dbt_databricks.current_catalog_sql": {"name": "current_catalog_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.current_catalog_sql", "macro_sql": "{% macro current_catalog_sql() %}\nSELECT current_catalog()\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044436, "supported_languages": null}, "macro.dbt_databricks.use_catalog": {"name": "use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.use_catalog", "macro_sql": "{% macro use_catalog(catalog) -%}\n {{ adapter.dispatch('use_catalog', 'dbt')(catalog) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__use_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04451, "supported_languages": null}, "macro.dbt_databricks.databricks__use_catalog": {"name": "databricks__use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__use_catalog", "macro_sql": "{% macro databricks__use_catalog(catalog) -%}\n {{ run_query_as(use_catalog_sql(catalog), 'use_catalog', fetch_result=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.use_catalog_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044729, "supported_languages": null}, "macro.dbt_databricks.use_catalog_sql": {"name": "use_catalog_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.use_catalog_sql", "macro_sql": "{% macro use_catalog_sql(catalog) %}\nUSE CATALOG {{ adapter.quote(catalog)|lower }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0448, "supported_languages": null}, "macro.dbt_databricks.databricks__py_write_table": {"name": "databricks__py_write_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.databricks__py_write_table", "macro_sql": "{% macro databricks__py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\nimport pyspark\n\n{{ py_try_import('pyspark.sql.connect.dataframe', 'newer_pyspark_available') }}\n{{ py_try_import('pandas', 'pandas_available') }}\n{{ py_try_import('pyspark.pandas', 'pyspark_pandas_api_available') }}\n{{ py_try_import('databricks.koalas', 'koalas_available') }}\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\n\nif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n if pyspark_pandas_api_available:\n try:\n df = pyspark.pandas.frame.DataFrame(df)\n except Exception as e:\n # If ANSI mode causes issues, fall back to spark.createDataFrame\n # This preserves the original pandas DataFrame for later conversion\n # Check for various ANSI mode related error messages\n error_str = str(e).lower()\n if any(ansi_error in error_str for ansi_error in [\n \"pandas_api_on_spark_fail_on_ansi_mode\",\n \"ansi mode\",\n \"ansimode\",\n \"sql_mode\",\n \"strict mode\"\n ]):\n pass # Will use spark.createDataFrame below\n else:\n raise e\n elif koalas_available:\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif newer_pyspark_available and isinstance(df, pyspark.sql.connect.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\nwriter = (\n df.write\n .mode(\"overwrite\")\n .option(\"overwriteSchema\", \"true\")\n{{ py_get_writer_options()|indent(8, True) }}\n)\n\nwriter.saveAsTable(\"{{ target_relation }}\")\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.py_try_import", "macro.dbt_databricks.py_get_writer_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.045419, "supported_languages": null}, "macro.dbt_databricks.py_get_writer_options": {"name": "py_get_writer_options", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_get_writer_options", "macro_sql": "\n\n{%- macro py_get_writer_options() -%}\n{%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n{%- set file_format = adapter.resolve_file_format(config) -%}\n{%- set partition_by = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n{%- set liquid_clustered_by = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set clustered_by = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n.format(\"{{ file_format }}\")\n{%- if location_root is not none %}\n{%- set model_path = adapter.compute_external_path(config, model, is_incremental()) %}\n.option(\"path\", \"{{ model_path }}\")\n{%- endif -%}\n{%- if partition_by is not none -%}\n {%- if partition_by is string -%}\n {%- set partition_by = [partition_by] -%}\n {%- endif %}\n.partitionBy({{ partition_by }})\n{%- endif -%}\n{%- if liquid_clustered_by and not is_incremental() -%}\n {%- if liquid_clustered_by is string -%}\n {%- set liquid_clustered_by = [liquid_clustered_by] -%}\n {%- endif %}\n.clusterBy({{ liquid_clustered_by }})\n{%- endif -%}\n{%- if (clustered_by is not none) and (buckets is not none) -%}\n {%- if clustered_by is string -%}\n {%- set clustered_by = [clustered_by] -%}\n {%- endif %}\n.bucketBy({{ buckets }}, {{ clustered_by }})\n{%- endif -%}\n{% endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.is_incremental"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.046102, "supported_languages": null}, "macro.dbt_databricks.py_try_import": {"name": "py_try_import", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_try_import", "macro_sql": "{% macro py_try_import(library, var_name) -%}\n# make sure {{ library }} exists before using it\ntry:\n import {{ library }}\n {{ var_name }} = True\nexcept ImportError:\n {{ var_name }} = False\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0461938, "supported_languages": null}, "macro.dbt_databricks.create_python_intermediate_table": {"name": "create_python_intermediate_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.create_python_intermediate_table", "macro_sql": "{% macro create_python_intermediate_table(relation, compiled_code) %}\n{{ compiled_code }}\n\n{%- set file_format = adapter.resolve_file_format(config) -%}\n\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\nimport pyspark\n\n{{ py_try_import('pyspark.sql.connect.dataframe', 'newer_pyspark_available') }}\n{{ py_try_import('pandas', 'pandas_available') }}\n{{ py_try_import('pyspark.pandas', 'pyspark_pandas_api_available') }}\n{{ py_try_import('databricks.koalas', 'koalas_available') }}\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\n\nif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n if pyspark_pandas_api_available:\n try:\n df = pyspark.pandas.frame.DataFrame(df)\n except Exception as e:\n # If ANSI mode causes issues, fall back to spark.createDataFrame\n # This preserves the original pandas DataFrame for later conversion\n # Check for various ANSI mode related error messages\n error_str = str(e).lower()\n if any(ansi_error in error_str for ansi_error in [\n \"pandas_api_on_spark_fail_on_ansi_mode\",\n \"ansi mode\",\n \"ansimode\",\n \"sql_mode\",\n \"strict mode\"\n ]):\n pass # Will use spark.createDataFrame below\n else:\n raise e\n elif koalas_available:\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif newer_pyspark_available and isinstance(df, pyspark.sql.connect.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\nwriter = (\n df.write\n .mode(\"overwrite\")\n .option(\"overwriteSchema\", \"true\")\n .format(\"{{ file_format }}\")\n)\n\nwriter.saveAsTable(\"{{ relation.render() }}\")\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.py_try_import"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04663, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_comment": {"name": "databricks__alter_column_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_comment", "macro_sql": "{% macro databricks__alter_column_comment(relation, column_dict) %}\n {% set file_format = adapter.resolve_file_format(config) %}\n {% if file_format in ['delta', 'hudi'] %}\n {% for column in column_dict.values() %}\n {% set comment = column['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set column_path = relation.render() ~ '.' ~ adapter.quote(column['name']) %}\n {{ run_query_as(comment_on_column_sql(column_path, escaped_comment), 'alter_column_comment', fetch_result=False) }}\n {% endfor %}\n {% else %}\n {{ log('WARNING - requested to update column comments, but file format ' ~ file_format ~ ' does not support that.') }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.comment_on_column_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0472262, "supported_languages": null}, "macro.dbt_databricks.comment_on_column_sql": {"name": "comment_on_column_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.comment_on_column_sql", "macro_sql": "{% macro comment_on_column_sql(column_path, escaped_comment) %}\n {%- if adapter.has_dbr_capability('comment_on_column') -%}\n COMMENT ON COLUMN {{ column_path }} IS '{{ escaped_comment }}'\n {%- else -%}\n {{ alter_table_change_column_comment_sql(column_path, escaped_comment) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.alter_table_change_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04735, "supported_languages": null}, "macro.dbt_databricks.alter_table_change_column_comment_sql": {"name": "alter_table_change_column_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_table_change_column_comment_sql", "macro_sql": "{% macro alter_table_change_column_comment_sql(column_path, escaped_comment) %}\n {%- set parts = column_path.split('.') -%}\n {%- if parts|length >= 4 -%}\n {%- set table_path = parts[:-1] | join('.') -%}\n {%- set column_name = parts[-1] -%}\n ALTER TABLE {{ table_path }} ALTER COLUMN {{ column_name }} COMMENT '{{ escaped_comment }}'\n {%- else -%}\n {{ exceptions.raise_compiler_error(\"Invalid column path: \" ~ column_path ~ \". Expected format: database.schema.table.column\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.047612, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_docs": {"name": "databricks__persist_docs", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__persist_docs", "macro_sql": "{% macro databricks__persist_docs(relation, model, for_relation, for_columns) -%}\n {%- if for_relation and config.persist_relation_docs() and model.description %}\n {{ run_query_as(alter_relation_comment_sql(relation, model.description), 'alter_relation_comment', fetch_result=False) }}\n {% endif %}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {%- set existing_columns = adapter.get_columns_in_relation(relation) -%}\n {%- set columns_to_persist_docs = adapter.get_persist_doc_columns(existing_columns, model.columns) -%}\n {{ alter_column_comment(relation, columns_to_persist_docs) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.alter_relation_comment_sql", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0480251, "supported_languages": null}, "macro.dbt_databricks.alter_relation_comment_sql": {"name": "alter_relation_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_relation_comment_sql", "macro_sql": "{% macro alter_relation_comment_sql(relation, description) %}\nCOMMENT ON {{ relation.type.render().upper() }} {{ relation.render() }} IS '{{ description | replace(\"'\", \"\\\\'\") }}'\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.048165, "supported_languages": null}, "macro.dbt_databricks.alter_column_comments": {"name": "alter_column_comments", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_column_comments", "macro_sql": "{% macro alter_column_comments(relation, column_dict) %}\n {% for column, comment in column_dict.items() %}\n {{ log('Updating comment for column ' ~ column ~ ' with comment ' ~ comment) }}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set column_path = relation.render() ~ '.' ~ adapter.quote(column) %}\n {{ run_query_as(comment_on_column_sql(column_path, escaped_comment), 'main', fetch_result=False) }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.comment_on_column_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.048413, "supported_languages": null}, "macro.dbt_databricks.databricks__list_relations_without_caching": {"name": "databricks__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__list_relations_without_caching", "macro_sql": "{% macro databricks__list_relations_without_caching(schema_relation) %}\n {{ return(adapter.get_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.048851, "supported_languages": null}, "macro.dbt_databricks.show_table_extended": {"name": "show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_table_extended", "macro_sql": "{% macro show_table_extended(schema_relation) %}\n {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_table_extended"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.048933, "supported_languages": null}, "macro.dbt_databricks.databricks__show_table_extended": {"name": "databricks__show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_table_extended", "macro_sql": "{% macro databricks__show_table_extended(schema_relation) %}\n {{ return(run_query_as(show_table_extended_sql(schema_relation), 'show_table_extended')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.show_table_extended_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049013, "supported_languages": null}, "macro.dbt_databricks.show_table_extended_sql": {"name": "show_table_extended_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_table_extended_sql", "macro_sql": "{% macro show_table_extended_sql(schema_relation) %}\nSHOW TABLE EXTENDED IN {{ schema_relation.without_identifier()|lower }} LIKE '{{ schema_relation.identifier|lower }}'\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049095, "supported_languages": null}, "macro.dbt_databricks.show_tables": {"name": "show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_tables", "macro_sql": "{% macro show_tables(relation) %}\n {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_tables"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049176, "supported_languages": null}, "macro.dbt_databricks.databricks__show_tables": {"name": "databricks__show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_tables", "macro_sql": "{% macro databricks__show_tables(relation) %}\n {{ return(run_query_as(show_tables_sql(relation), 'show_tables')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.show_tables_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049251, "supported_languages": null}, "macro.dbt_databricks.show_tables_sql": {"name": "show_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_tables_sql", "macro_sql": "{% macro show_tables_sql(relation) %}\n\nSHOW TABLES IN {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0493052, "supported_languages": null}, "macro.dbt_databricks.databricks__list_schemas": {"name": "databricks__list_schemas", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__list_schemas", "macro_sql": "{% macro databricks__list_schemas(database) -%}\n {{ return(run_query_as(list_schemas_sql(database), 'list_schemas')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.list_schemas_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049377, "supported_languages": null}, "macro.dbt_databricks.list_schemas_sql": {"name": "list_schemas_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.list_schemas_sql", "macro_sql": "{% macro list_schemas_sql(database) %}\n{% if database %}\n SHOW SCHEMAS IN {{ adapter.quote(database) }}\n{% else %}\n SHOW SCHEMAS\n{% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049463, "supported_languages": null}, "macro.dbt_databricks.databricks__check_schema_exists": {"name": "databricks__check_schema_exists", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__check_schema_exists", "macro_sql": "{% macro databricks__check_schema_exists(database, schema) %}\n {{ return(run_query_as(check_schema_exists_sql(database, schema), 'check_schema_exists')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.check_schema_exists_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0495481, "supported_languages": null}, "macro.dbt_databricks.check_schema_exists_sql": {"name": "check_schema_exists_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.check_schema_exists_sql", "macro_sql": "{% macro check_schema_exists_sql(database, schema) %}\n SHOW SCHEMAS IN {{ adapter.quote(database) }} LIKE '{{ schema }}'\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049621, "supported_languages": null}, "macro.dbt_databricks.show_views": {"name": "show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_views", "macro_sql": "{% macro show_views(relation) %}\n {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_views"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04978, "supported_languages": null}, "macro.dbt_databricks.databricks__show_views": {"name": "databricks__show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_views", "macro_sql": "{% macro databricks__show_views(relation) %}\n {{ return(run_query_as(show_views_sql(relation), 'show_views')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.show_views_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049862, "supported_languages": null}, "macro.dbt_databricks.show_views_sql": {"name": "show_views_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_views_sql", "macro_sql": "{% macro show_views_sql(relation) %}\nSHOW VIEWS IN {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0499241, "supported_languages": null}, "macro.dbt_databricks.databricks__get_relation_last_modified": {"name": "databricks__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__get_relation_last_modified", "macro_sql": "{% macro databricks__get_relation_last_modified(information_schema, relations) -%}\n {% call statement('last_modified', fetch_result=True) %}\n {{ get_relation_last_modified_sql(information_schema, relations) }}\n {% endcall %}\n {{ return(load_result('last_modified')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_relation_last_modified_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.05005, "supported_languages": null}, "macro.dbt_databricks.get_relation_last_modified_sql": {"name": "get_relation_last_modified_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_relation_last_modified_sql", "macro_sql": "{% macro get_relation_last_modified_sql(information_schema, relations) %}\n {% if information_schema.is_hive_metastore() %}\n {%- for relation in relations -%}\nSELECT\n '{{ relation.schema }}' AS schema,\n '{{ relation.identifier }}' AS identifier,\n max(timestamp) AS last_modified,\n {{ current_timestamp() }} AS snapshotted_at\n FROM (DESCRIBE HISTORY {{ relation.render() }})\n {% if not loop.last %}\nUNION ALL\n {% endif %}\n {%- endfor -%}\n {% else %}\nSELECT\n table_schema AS schema,\n table_name AS identifier,\n last_altered AS last_modified,\n {{ current_timestamp() }} AS snapshotted_at\nFROM `system`.`information_schema`.`tables`\nWHERE table_catalog = '{{ information_schema.database|lower }}'\n AND (\n {%- for relation in relations -%}\n (table_schema = '{{ relation.schema|lower }}' AND\n table_name = '{{ relation.identifier|lower }}'){%- if not loop.last %} OR {% endif -%}\n {%- endfor -%}\n )\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0503528, "supported_languages": null}, "macro.dbt_databricks.get_view_description": {"name": "get_view_description", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_view_description", "macro_sql": "{% macro get_view_description(relation) %}\n {{ return(run_query_as(get_view_description_sql(relation), 'get_view_description')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.get_view_description_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.050446, "supported_languages": null}, "macro.dbt_databricks.get_view_description_sql": {"name": "get_view_description_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_view_description_sql", "macro_sql": "{% macro get_view_description_sql(relation) %}\nSELECT *\nFROM `system`.`information_schema`.`views`\nWHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0505338, "supported_languages": null}, "macro.dbt_databricks.get_uc_tables": {"name": "get_uc_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_uc_tables", "macro_sql": "{% macro get_uc_tables(relation) %}\n {{ return(run_query_as(get_uc_tables_sql(relation), 'get_uc_tables')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.get_uc_tables_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0506089, "supported_languages": null}, "macro.dbt_databricks.get_uc_tables_sql": {"name": "get_uc_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_uc_tables_sql", "macro_sql": "{% macro get_uc_tables_sql(relation) %}\nSELECT\n table_name,\n if(table_type IN ('EXTERNAL', 'MANAGED', 'MANAGED_SHALLOW_CLONE', 'EXTERNAL_SHALLOW_CLONE'), 'table', lower(table_type)) AS table_type,\n lower(data_source_format) AS file_format,\n table_owner,\n if(\n table_type IN (\n 'EXTERNAL',\n 'MANAGED',\n 'MANAGED_SHALLOW_CLONE',\n 'EXTERNAL_SHALLOW_CLONE'\n ),\n lower(table_type),\n NULL\n ) AS databricks_table_type\nFROM `system`.`information_schema`.`tables`\nWHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n {%- if relation.identifier %}\n AND table_name = '{{ relation.identifier|lower }}'\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.050743, "supported_languages": null}, "macro.dbt_databricks.describe_table_extended_as_json": {"name": "describe_table_extended_as_json", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.describe_table_extended_as_json", "macro_sql": "{% macro describe_table_extended_as_json(relation) %}\n {{ return(run_query_as(describe_table_extended_as_json_sql(relation), 'describe_table_extended_as_json')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.describe_table_extended_as_json_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.050828, "supported_languages": null}, "macro.dbt_databricks.describe_table_extended_as_json_sql": {"name": "describe_table_extended_as_json_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.describe_table_extended_as_json_sql", "macro_sql": "{% macro describe_table_extended_as_json_sql(relation) %}\nDESCRIBE TABLE EXTENDED {{ relation.render() }} AS JSON\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.050889, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments": {"name": "get_columns_comments", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.get_columns_comments", "macro_sql": "{% macro get_columns_comments(relation) -%}\n {{ return(run_query_as(get_columns_comments_sql(relation), 'get_columns_comments')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.get_columns_comments_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051106, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments_sql": {"name": "get_columns_comments_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.get_columns_comments_sql", "macro_sql": "{% macro get_columns_comments_sql(relation) %}\nDESCRIBE TABLE {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051162, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments_as_json": {"name": "get_columns_comments_as_json", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.get_columns_comments_as_json", "macro_sql": "{% macro get_columns_comments_as_json(relation) -%}\n {{ return(run_query_as(get_columns_comments_as_json_sql(relation), 'get_columns_comments_as_json')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.get_columns_comments_as_json_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0512462, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments_as_json_sql": {"name": "get_columns_comments_as_json_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.get_columns_comments_as_json_sql", "macro_sql": "{% macro get_columns_comments_as_json_sql(relation) %}\n DESCRIBE TABLE EXTENDED {{ relation.render() }} AS JSON\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051308, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_relation_add_remove_columns": {"name": "databricks__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.databricks__alter_relation_add_remove_columns", "macro_sql": "{% macro databricks__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n {% if remove_columns %}\n {{ run_query_as(drop_columns_sql(relation, remove_columns), 'alter_relation_remove_columns', fetch_result=False) }}\n {% endif %}\n\n {% if add_columns %}\n {{ run_query_as(add_columns_sql(relation, add_columns), 'alter_relation_add_columns', fetch_result=False) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.drop_columns_sql", "macro.dbt_databricks.add_columns_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0514932, "supported_languages": null}, "macro.dbt_databricks.drop_columns_sql": {"name": "drop_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.drop_columns_sql", "macro_sql": "{% macro drop_columns_sql(relation, remove_columns) %}\nALTER TABLE {{ relation.render() }} DROP COLUMNS ({{ api.Column.format_remove_column_list(remove_columns) }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051583, "supported_languages": null}, "macro.dbt_databricks.add_columns_sql": {"name": "add_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.add_columns_sql", "macro_sql": "{% macro add_columns_sql(relation, add_columns) %}\nALTER TABLE {{ relation.render() }} ADD COLUMNS ({{ api.Column.format_add_column_list(add_columns) }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0516748, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_type": {"name": "databricks__alter_column_type", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_type", "macro_sql": "{% macro databricks__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ adapter.quote(column_name) }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051799, "supported_languages": null}, "macro.dbt_spark.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() %}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.053226, "supported_languages": null}, "macro.dbt_spark.spark__tblproperties_clause": {"name": "spark__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__tblproperties_clause", "macro_sql": "{% macro spark__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.053417, "supported_languages": null}, "macro.dbt_spark.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {{ return(adapter.dispatch('file_format_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__file_format_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.05351, "supported_languages": null}, "macro.dbt_spark.spark__file_format_clause": {"name": "spark__file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__file_format_clause", "macro_sql": "{% macro spark__file_format_clause() %}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n {%- if file_format is not none %}\n using {{ file_format }}\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0536478, "supported_languages": null}, "macro.dbt_spark.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.location_clause", "macro_sql": "{% macro location_clause() %}\n {{ return(adapter.dispatch('location_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__location_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.053788, "supported_languages": null}, "macro.dbt_spark.spark__location_clause": {"name": "spark__location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__location_clause", "macro_sql": "{% macro spark__location_clause() %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.053953, "supported_languages": null}, "macro.dbt_spark.options_clause": {"name": "options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.options_clause", "macro_sql": "{% macro options_clause() -%}\n {{ return(adapter.dispatch('options_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__options_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.054026, "supported_languages": null}, "macro.dbt_spark.spark__options_clause": {"name": "spark__options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__options_clause", "macro_sql": "{% macro spark__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.054511, "supported_languages": null}, "macro.dbt_spark.comment_clause": {"name": "comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.comment_clause", "macro_sql": "{% macro comment_clause() %}\n {{ return(adapter.dispatch('comment_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__comment_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.054594, "supported_languages": null}, "macro.dbt_spark.spark__comment_clause": {"name": "spark__comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__comment_clause", "macro_sql": "{% macro spark__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endif %}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.054822, "supported_languages": null}, "macro.dbt_spark.partition_cols": {"name": "partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.partition_cols", "macro_sql": "{% macro partition_cols(label, required=false) %}\n {{ return(adapter.dispatch('partition_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0549371, "supported_languages": null}, "macro.dbt_spark.spark__partition_cols": {"name": "spark__partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__partition_cols", "macro_sql": "{% macro spark__partition_cols(label, required=false) %}\n {%- set cols = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055181, "supported_languages": null}, "macro.dbt_spark.clustered_cols": {"name": "clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.clustered_cols", "macro_sql": "{% macro clustered_cols(label, required=false) %}\n {{ return(adapter.dispatch('clustered_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__clustered_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0552902, "supported_languages": null}, "macro.dbt_spark.spark__clustered_cols": {"name": "spark__clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__clustered_cols", "macro_sql": "{% macro spark__clustered_cols(label, required=false) %}\n {%- set cols = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n {%- if (cols is not none) and (buckets is not none) %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n ) into {{ buckets }} buckets\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055617, "supported_languages": null}, "macro.dbt_spark.fetch_tbl_properties": {"name": "fetch_tbl_properties", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.fetch_tbl_properties", "macro_sql": "{% macro fetch_tbl_properties(relation) -%}\n {% call statement('list_properties', fetch_result=True) -%}\n SHOW TBLPROPERTIES {{ relation }}\n {% endcall %}\n {% do return(load_result('list_properties').table) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055747, "supported_languages": null}, "macro.dbt_spark.create_temporary_view": {"name": "create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.create_temporary_view", "macro_sql": "{% macro create_temporary_view(relation, compiled_code) -%}\n {{ return(adapter.dispatch('create_temporary_view', 'dbt')(relation, compiled_code)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__create_temporary_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055846, "supported_languages": null}, "macro.dbt_spark.spark__create_temporary_view": {"name": "spark__create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_temporary_view", "macro_sql": "{% macro spark__create_temporary_view(relation, compiled_code) -%}\n create or replace temporary view {{ relation }} as\n {{ compiled_code }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055918, "supported_languages": null}, "macro.dbt_spark.spark__create_table_as": {"name": "spark__create_table_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_table_as", "macro_sql": "{%- macro spark__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'iceberg'] %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ tblproperties_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_spark.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.tblproperties_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_spark.py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.056461, "supported_languages": null}, "macro.dbt_spark.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.056577, "supported_languages": null}, "macro.dbt_spark.spark__persist_constraints": {"name": "spark__persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_constraints", "macro_sql": "{% macro spark__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config.enforced and config.get('file_format', 'delta') == 'delta' %}\n {% do alter_table_add_constraints(relation, model.constraints) %}\n {% do alter_column_set_constraints(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.alter_table_add_constraints", "macro.dbt_spark.alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.056758, "supported_languages": null}, "macro.dbt_spark.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0568519, "supported_languages": null}, "macro.dbt_spark.spark__alter_table_add_constraints": {"name": "spark__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_table_add_constraints", "macro_sql": "{% macro spark__alter_table_add_constraints(relation, constraints) %}\n {% for constraint in constraints %}\n {% if constraint.type == 'check' and not is_incremental() %}\n {%- set constraint_hash = local_md5(column_name ~ \";\" ~ constraint.expression ~ \";\" ~ loop.index) -%}\n {% call statement() %}\n alter table {{ relation }} add constraint {{ constraint.name if constraint.name else constraint_hash }} check ({{ constraint.expression }});\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0570781, "supported_languages": null}, "macro.dbt_spark.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.057175, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_set_constraints": {"name": "spark__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_set_constraints", "macro_sql": "{% macro spark__alter_column_set_constraints(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set constraints = column_dict[column_name]['constraints'] %}\n {% for constraint in constraints %}\n {% if constraint.type != 'not_null' %}\n {{ exceptions.warn('Invalid constraint for column ' ~ column_name ~ '. Only `not_null` is supported.') }}\n {% else %}\n {% set quoted_name = adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name %}\n {% call statement() %}\n alter table {{ relation }} change column {{ quoted_name }} set not null {{ constraint.expression or \"\" }};\n {% endcall %}\n {% endif %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0574908, "supported_languages": null}, "macro.dbt_spark.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.057767, "supported_languages": null}, "macro.dbt_spark.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.057884, "supported_languages": null}, "macro.dbt_spark.spark__create_view_as": {"name": "spark__create_view_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_view_as", "macro_sql": "{% macro spark__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_spark.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.05811, "supported_languages": null}, "macro.dbt_spark.spark__create_schema": {"name": "spark__create_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_schema", "macro_sql": "{% macro spark__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{relation}}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058201, "supported_languages": null}, "macro.dbt_spark.spark__drop_schema": {"name": "spark__drop_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_schema", "macro_sql": "{% macro spark__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058292, "supported_languages": null}, "macro.dbt_spark.get_columns_in_relation_raw": {"name": "get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_columns_in_relation_raw", "macro_sql": "{% macro get_columns_in_relation_raw(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation_raw', 'dbt')(relation)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation_raw"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058371, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation_raw": {"name": "spark__get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation_raw", "macro_sql": "{% macro spark__get_columns_in_relation_raw(relation) -%}\n {% call statement('get_columns_in_relation_raw', fetch_result=True) %}\n describe extended {{ relation }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation_raw').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058483, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation": {"name": "spark__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation", "macro_sql": "{% macro spark__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n describe extended {{ relation.include(schema=(schema is not none)) }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058639, "supported_languages": null}, "macro.dbt_spark.spark__list_relations_without_caching": {"name": "spark__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_relations_without_caching", "macro_sql": "{% macro spark__list_relations_without_caching(relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n show table extended in {{ relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058775, "supported_languages": null}, "macro.dbt_spark.list_relations_show_tables_without_caching": {"name": "list_relations_show_tables_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.list_relations_show_tables_without_caching", "macro_sql": "{% macro list_relations_show_tables_without_caching(schema_relation) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('list_relations_without_caching_show_tables', fetch_result=True) -%}\n show tables in {{ schema_relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching_show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058906, "supported_languages": null}, "macro.dbt_spark.describe_table_extended_without_caching": {"name": "describe_table_extended_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.describe_table_extended_without_caching", "macro_sql": "{% macro describe_table_extended_without_caching(table_name) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('describe_table_extended_without_caching', fetch_result=True) -%}\n describe extended {{ table_name }}\n {% endcall %}\n {% do return(load_result('describe_table_extended_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.059037, "supported_languages": null}, "macro.dbt_spark.spark__list_schemas": {"name": "spark__list_schemas", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_schemas", "macro_sql": "{% macro spark__list_schemas(database) -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n show databases\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0591629, "supported_languages": null}, "macro.dbt_spark.spark__rename_relation": {"name": "spark__rename_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__rename_relation", "macro_sql": "{% macro spark__rename_relation(from_relation, to_relation) -%}\n {% call statement('rename_relation') -%}\n {% if not from_relation.type %}\n {% do exceptions.raise_database_error(\"Cannot rename a relation with a blank type: \" ~ from_relation.identifier) %}\n {% elif from_relation.type in ('table') %}\n alter table {{ from_relation }} rename to {{ to_relation }}\n {% elif from_relation.type == 'view' %}\n alter view {{ from_relation }} rename to {{ to_relation }}\n {% else %}\n {% do exceptions.raise_database_error(\"Unknown type '\" ~ from_relation.type ~ \"' for relation: \" ~ from_relation.identifier) %}\n {% endif %}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0594292, "supported_languages": null}, "macro.dbt_spark.spark__drop_relation": {"name": "spark__drop_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_relation", "macro_sql": "{% macro spark__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.059542, "supported_languages": null}, "macro.dbt_spark.spark__generate_database_name": {"name": "spark__generate_database_name", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__generate_database_name", "macro_sql": "{% macro spark__generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(None) %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.059618, "supported_languages": null}, "macro.dbt_spark.spark__persist_docs": {"name": "spark__persist_docs", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_docs", "macro_sql": "{% macro spark__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% set filtered_columns = validate_doc_columns(relation, model.columns, existing_columns) %}\n {% do alter_column_comment(relation, filtered_columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.validate_doc_columns", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.059833, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_comment": {"name": "spark__alter_column_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_comment", "macro_sql": "{% macro spark__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'hudi', 'iceberg'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n {% if relation.is_iceberg %}\n alter table {{ relation }} alter column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% else %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endif %}\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.060274, "supported_languages": null}, "macro.dbt_spark.spark__make_temp_relation": {"name": "spark__make_temp_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__make_temp_relation", "macro_sql": "{% macro spark__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(path = {\n \"identifier\": tmp_identifier\n }) -%}\n\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.060457, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_type": {"name": "spark__alter_column_type", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_type", "macro_sql": "{% macro spark__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ column_name }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.060568, "supported_languages": null}, "macro.dbt_spark.spark__alter_relation_add_remove_columns": {"name": "spark__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_relation_add_remove_columns", "macro_sql": "{% macro spark__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if remove_columns %}\n {% if relation.is_delta %}\n {% set platform_name = 'Delta Lake' %}\n {% elif relation.is_iceberg %}\n {% set platform_name = 'Iceberg' %}\n {% else %}\n {% set platform_name = 'Apache Spark' %}\n {% endif %}\n {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}\n {% endif %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% if add_columns %} add columns {% endif %}\n {% for column in add_columns %}\n {{ column.quoted }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0609531, "supported_languages": null}, "macro.dbt_spark.spark__copy_grants": {"name": "spark__copy_grants", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__copy_grants", "macro_sql": "{% macro spark__copy_grants() %}\n\n {% if config.materialized == 'view' %}\n {#-- Spark views don't copy grants when they're replaced --#}\n {{ return(False) }}\n\n {% else %}\n {#-- This depends on how we're replacing the table, which depends on its file format\n -- Just play it safe by assuming that grants have been copied over, and need to be checked / possibly revoked\n -- We can make this more efficient in the future\n #}\n {{ return(True) }}\n\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0612109, "supported_languages": null}, "macro.dbt_spark.spark__get_grant_sql": {"name": "spark__get_grant_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_grant_sql", "macro_sql": "\n\n\n{%- macro spark__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.061316, "supported_languages": null}, "macro.dbt_spark.spark__get_revoke_sql": {"name": "spark__get_revoke_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_revoke_sql", "macro_sql": "\n\n\n{%- macro spark__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.061407, "supported_languages": null}, "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement": {"name": "spark__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n\n{%- macro spark__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(False) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.061465, "supported_languages": null}, "macro.dbt_spark.spark__call_dcl_statements": {"name": "spark__call_dcl_statements", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__call_dcl_statements", "macro_sql": "{% macro spark__call_dcl_statements(dcl_statement_list) %}\n {% for dcl_statement in dcl_statement_list %}\n {% call statement('grant_or_revoke') %}\n {{ dcl_statement }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0615768, "supported_languages": null}, "macro.dbt_spark.spark__get_binding_char": {"name": "spark__get_binding_char", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__get_binding_char", "macro_sql": "{% macro spark__get_binding_char() %}\n {{ return('?' if target.method == 'odbc' else '%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.062026, "supported_languages": null}, "macro.dbt_spark.spark__reset_csv_table": {"name": "spark__reset_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__reset_csv_table", "macro_sql": "{% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {{ adapter.drop_relation(old_relation) }}\n {% endif %}\n {% set sql = create_csv_table(model, agate_table) %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.06225, "supported_languages": null}, "macro.dbt_spark.spark__load_csv_rows": {"name": "spark__load_csv_rows", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__load_csv_rows", "macro_sql": "{% macro spark__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.062837, "supported_languages": null}, "macro.dbt_spark.spark__create_csv_table": {"name": "spark__create_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__create_csv_table", "macro_sql": "{% macro spark__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.063313, "supported_languages": null}, "macro.dbt_spark.materialization_view_spark": {"name": "materialization_view_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_spark.materialization_view_spark", "macro_sql": "{% materialization view, adapter='spark' -%}\n {{ return(create_or_replace_view()) }}\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.create_or_replace_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0634181, "supported_languages": ["sql"]}, "macro.dbt_spark.materialization_table_spark": {"name": "materialization_table_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.materialization_table_spark", "macro_sql": "{% materialization table, adapter = 'spark', supported_languages=['sql', 'python'] %}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation is not none %}\n {% set is_delta = (old_relation.is_delta and config.get('file_format', validator=validation.any[basestring]) == 'delta') %}\n {% set is_iceberg = (old_relation.is_iceberg and config.get('file_format', validator=validation.any[basestring]) == 'iceberg') %}\n {% set old_relation_type = old_relation.type %}\n {% else %}\n {% set is_delta = false %}\n {% set is_iceberg = false %}\n {% set old_relation_type = target_relation.type %}\n {% endif %}\n\n {% if not is_delta and not is_iceberg %}\n {% set existing_relation = target_relation %}\n {{ adapter.drop_relation(existing_relation.incorporate(type=old_relation_type)) }}\n {% endif %}\n\n -- build model\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt_spark.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.064458, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.py_write_table": {"name": "py_write_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_write_table", "macro_sql": "{% macro py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\n# make sure pyspark exists in the namepace, for 7.3.x-scala2.12 it does not exist\nimport pyspark\n# make sure pandas exists before using it\ntry:\n import pandas\n pandas_available = True\nexcept ImportError:\n pandas_available = False\n\n# make sure pyspark.pandas exists before using it\ntry:\n import pyspark.pandas\n pyspark_pandas_api_available = True\nexcept ImportError:\n pyspark_pandas_api_available = False\n\n# make sure databricks.koalas exists before using it\ntry:\n import databricks.koalas\n koalas_available = True\nexcept ImportError:\n koalas_available = False\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\nif pyspark_pandas_api_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = pyspark.pandas.frame.DataFrame(df)\nelif koalas_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\ndf.write.mode(\"overwrite\").format(\"{{ config.get('file_format', 'delta') }}\").option(\"overwriteSchema\", \"true\").saveAsTable(\"{{ target_relation }}\")\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0646248, "supported_languages": null}, "macro.dbt_spark.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n# how to execute python model in notebook\n# dbt = dbtObj(spark.table)\n# df = model(dbt, spark)\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.064667, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_hash_arguments": {"name": "spark__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_hash_arguments", "macro_sql": "{% macro spark__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as string ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.065334, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_string_as_time": {"name": "spark__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_string_as_time", "macro_sql": "{% macro spark__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"to_timestamp('\" ~ timestamp ~ \"')\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.065428, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_merge_sql": {"name": "spark__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_merge_sql", "macro_sql": "{% macro spark__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {% if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {% else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {% endif %}\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n when matched\n {% if config.get(\"dbt_valid_to_current\") %}\n and ( DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or\n DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null )\n {% else %}\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n {% endif %}\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert *\n ;\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.065749, "supported_languages": null}, "macro.dbt_spark.spark_build_snapshot_staging_table": {"name": "spark_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark_build_snapshot_staging_table", "macro_sql": "{% macro spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {% if target_relation.is_iceberg %}\n {# iceberg catalog does not support create view, but regular spark does. We removed the catalog and schema #}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=none,\n database=none,\n type='view') -%}\n {% else %}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=none,\n type='view') -%}\n {% endif %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_view_as(tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.066113, "supported_languages": null}, "macro.dbt_spark.spark__post_snapshot": {"name": "spark__post_snapshot", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__post_snapshot", "macro_sql": "{% macro spark__post_snapshot(staging_relation) %}\n {% do adapter.drop_relation(staging_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.066187, "supported_languages": null}, "macro.dbt_spark.spark__create_columns": {"name": "spark__create_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__create_columns", "macro_sql": "{% macro spark__create_columns(relation, columns) %}\n {% if columns|length > 0 %}\n {% call statement() %}\n alter table {{ relation }} add columns (\n {% for column in columns %}\n `{{ column.name }}` {{ column.data_type }} {{- ',' if not loop.last -}}\n {% endfor %}\n );\n {% endcall %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0664508, "supported_languages": null}, "macro.dbt_spark.materialization_snapshot_spark": {"name": "materialization_snapshot_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.materialization_snapshot_spark", "macro_sql": "{% materialization snapshot, adapter='spark' %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format') or 'parquet' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=none,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if file_format not in ['delta', 'iceberg', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_iceberg and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.schema) %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.valid_snapshot_target(target_relation, columns) }}\n\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.create_schema", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.068236, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__can_clone_table": {"name": "spark__can_clone_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__can_clone_table", "macro_sql": "{% macro spark__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.068533, "supported_languages": null}, "macro.dbt_spark.spark__create_or_replace_clone": {"name": "spark__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__create_or_replace_clone", "macro_sql": "{% macro spark__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0686028, "supported_languages": null}, "macro.dbt_spark.materialization_clone_spark": {"name": "materialization_clone_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.materialization_clone_spark", "macro_sql": "\n\n{%- materialization clone, adapter='spark' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if file_format != 'delta' -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n shallow clone requires file_format be set to 'delta'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {%- elif other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.06957, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__get_merge_update_columns": {"name": "spark__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/column_helpers.sql", "original_file_path": "macros/materializations/incremental/column_helpers.sql", "unique_id": "macro.dbt_spark.spark__get_merge_update_columns", "macro_sql": "{% macro spark__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.070068, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_file_format": {"name": "dbt_spark_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_file_format", "macro_sql": "{% macro dbt_spark_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'iceberg', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.07071, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy": {"name": "dbt_spark_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'append', 'merge', 'insert_overwrite', 'microbatch'\n {%- endset %}\n\n {% set invalid_merge_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via endpoint\n Use the 'append' or 'merge' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite', 'microbatch'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'iceberg', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_merge_msg) %}\n {% endif %}\n {% if raw_strategy in ['insert_overwrite', 'microbatch'] and target.endpoint %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_endpoint_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.071131, "supported_languages": null}, "macro.dbt_spark.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation, existing_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n {% if existing_relation.is_iceberg %}\n {# removed table from statement for iceberg #}\n insert overwrite {{ target_relation }}\n {# removed partition_cols for iceberg as well #}\n {% else %}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {% endif %}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0716958, "supported_languages": null}, "macro.dbt_spark.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert into table {{ target_relation }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.071851, "supported_languages": null}, "macro.dbt_spark.spark__get_merge_sql": {"name": "spark__get_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.spark__get_merge_sql", "macro_sql": "{% macro spark__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n when matched then update set\n {% if update_columns -%}{%- for column_name in update_columns %}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {%- else %} * {% endif %}\n\n when not matched then insert *\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.07249, "supported_languages": null}, "macro.dbt_spark.dbt_spark_get_incremental_sql": {"name": "dbt_spark_get_incremental_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro_sql": "{% macro dbt_spark_get_incremental_sql(strategy, source, target, existing, unique_key, incremental_predicates) %}\n {%- if strategy == 'append' -%}\n {#-- insert new records into existing table, without updating or overwriting #}\n {{ get_insert_into_sql(source, target) }}\n {%- elif strategy == 'insert_overwrite' -%}\n {#-- insert statements don't like CTEs, so support them via a temp view #}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'microbatch' -%}\n {#-- microbatch wraps insert_overwrite, and requires a partition_by config #}\n {% set missing_partition_key_microbatch_msg -%}\n dbt-spark 'microbatch' incremental strategy requires a `partition_by` config.\n Ensure you are using a `partition_by` column that is of grain {{ config.get('batch_size') }}.\n {%- endset %}\n\n {%- if not config.get('partition_by') -%}\n {{ exceptions.raise_compiler_error(missing_partition_key_microbatch_msg) }}\n {%- endif -%}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'merge' -%}\n {#-- merge all columns for datasources which implement MERGE INTO (e.g. databricks, iceberg) - schema changes are handled for us #}\n {{ get_merge_sql(target, source, unique_key, dest_columns=none, incremental_predicates=incremental_predicates) }}\n {%- else -%}\n {% set no_sql_for_strategy_msg -%}\n No known SQL for the incremental strategy provided: {{ strategy }}\n {%- endset %}\n {%- do exceptions.raise_compiler_error(no_sql_for_strategy_msg) -%}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_insert_into_sql", "macro.dbt_spark.get_insert_overwrite_sql", "macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0729558, "supported_languages": null}, "macro.dbt_spark.materialization_incremental_spark": {"name": "materialization_incremental_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_spark.materialization_incremental_spark", "macro_sql": "{% materialization incremental, adapter='spark', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='parquet') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'append' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set file_format = dbt_spark_validate_get_file_format(raw_file_format) -%}\n {%- set strategy = dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}\n {%- set target_relation = this -%}\n {%- set existing_relation = load_relation(this) -%}\n {% set tmp_relation = this.incorporate(path = {\"identifier\": this.identifier ~ '__dbt_tmp'}) -%}\n\n {#-- for SQL model we will create temp view that doesn't have database and schema --#}\n {%- if language == 'sql'-%}\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode --#}\n {%- if strategy in ['insert_overwrite', 'microbatch'] and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- elif existing_relation.is_view or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- else -%}\n {#-- Relation must be merged --#}\n {%- call statement('create_tmp_relation', language=language) -%}\n {{ create_table_as(True, tmp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, tmp_relation, existing_relation) -%}\n {%- call statement('main') -%}\n {{ dbt_spark_get_incremental_sql(strategy, tmp_relation, target_relation, existing_relation, unique_key, incremental_predicates) }}\n {%- endcall -%}\n {%- if language == 'python' -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why do neither drop_relation or adapter.drop_relation work here?!\n --#}\n {% call statement('drop_relation') -%}\n drop table if exists {{ tmp_relation }}\n {%- endcall %}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_spark.dbt_spark_validate_get_file_format", "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.load_relation", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_spark.persist_constraints", "macro.dbt.should_full_refresh", "macro.dbt.process_schema_changes", "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0746171, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.spark__date": {"name": "spark__date", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt_spark.spark__date", "macro_sql": "{% macro spark__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'yyyy-MM-dd')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.074813, "supported_languages": null}, "macro.dbt_spark.spark__concat": {"name": "spark__concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt_spark.spark__concat", "macro_sql": "{% macro spark__concat(fields) -%}\n concat({{ fields|join(', ') }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0749052, "supported_languages": null}, "macro.dbt_spark.assert_not_null": {"name": "assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.assert_not_null", "macro_sql": "{% macro assert_not_null(function, arg) -%}\n {{ return(adapter.dispatch('assert_not_null', 'dbt')(function, arg)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.075047, "supported_languages": null}, "macro.dbt_spark.spark__assert_not_null": {"name": "spark__assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.spark__assert_not_null", "macro_sql": "{% macro spark__assert_not_null(function, arg) %}\n\n coalesce({{function}}({{arg}}), nvl2({{function}}({{arg}}), assert_true({{function}}({{arg}}) is not null), null))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.075142, "supported_languages": null}, "macro.dbt_spark.spark__dateadd": {"name": "spark__dateadd", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_spark.spark__dateadd", "macro_sql": "{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {%- set clock_component -%}\n {# make sure the dates + timestamps are real, otherwise raise an error asap #}\n to_unix_timestamp({{ assert_not_null('to_timestamp', from_date_or_timestamp) }})\n - to_unix_timestamp({{ assert_not_null('date', from_date_or_timestamp) }})\n {%- endset -%}\n\n {%- if datepart in ['day', 'week'] -%}\n\n {%- set multiplier = 7 if datepart == 'week' else 1 -%}\n\n to_timestamp(\n to_unix_timestamp(\n date_add(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ['month', 'quarter', 'year'] -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'month' -%} 1\n {%- elif datepart == 'quarter' -%} 3\n {%- elif datepart == 'year' -%} 12\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp(\n add_months(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n {{ assert_not_null('to_unix_timestamp', from_date_or_timestamp) }}\n + cast({{interval}} * {{multiplier}} as int)\n )\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro dateadd not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.07609, "supported_languages": null}, "macro.dbt_spark.spark__current_timestamp": {"name": "spark__current_timestamp", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/timestamps.sql", "original_file_path": "macros/utils/timestamps.sql", "unique_id": "macro.dbt_spark.spark__current_timestamp", "macro_sql": "{% macro spark__current_timestamp() -%}\n current_timestamp()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.07618, "supported_languages": null}, "macro.dbt_spark.spark__escape_single_quotes": {"name": "spark__escape_single_quotes", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt_spark.spark__escape_single_quotes", "macro_sql": "{% macro spark__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"\\\\'\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.076289, "supported_languages": null}, "macro.dbt_spark.spark__listagg": {"name": "spark__listagg", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_spark.spark__listagg", "macro_sql": "{% macro spark__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if order_by_clause %}\n {{ exceptions.warn(\"order_by_clause is not supported for listagg on Spark/Databricks\") }}\n {% endif %}\n\n {% set collect_list %} collect_list({{ measure }}) {% endset %}\n\n {% set limited %} slice({{ collect_list }}, 1, {{ limit_num }}) {% endset %}\n\n {% set collected = limited if limit_num else collect_list %}\n\n {% set final %} array_join({{ collected }}, {{ delimiter_text }}) {% endset %}\n\n {% do return(final) %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0765948, "supported_languages": null}, "macro.dbt_spark.spark__datediff": {"name": "spark__datediff", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_spark.spark__datediff", "macro_sql": "{% macro spark__datediff(first_date, second_date, datepart) %}\n\n {%- if datepart in ['day', 'week', 'month', 'quarter', 'year'] -%}\n\n {# make sure the dates are real, otherwise raise an error asap #}\n {% set first_date = assert_not_null('date', first_date) %}\n {% set second_date = assert_not_null('date', second_date) %}\n\n {%- endif -%}\n\n {%- if datepart == 'day' -%}\n\n datediff({{second_date}}, {{first_date}})\n\n {%- elif datepart == 'week' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(datediff({{second_date}}, {{first_date}})/7)\n else ceil(datediff({{second_date}}, {{first_date}})/7)\n end\n\n -- did we cross a week boundary (Sunday)?\n + case\n when {{first_date}} < {{second_date}} and dayofweek({{second_date}}) < dayofweek({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofweek({{second_date}}) > dayofweek({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'month' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}})))\n else ceil(months_between(date({{second_date}}), date({{first_date}})))\n end\n\n -- did we cross a month boundary?\n + case\n when {{first_date}} < {{second_date}} and dayofmonth({{second_date}}) < dayofmonth({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofmonth({{second_date}}) > dayofmonth({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'quarter' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}}))/3)\n else ceil(months_between(date({{second_date}}), date({{first_date}}))/3)\n end\n\n -- did we cross a quarter boundary?\n + case\n when {{first_date}} < {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n < (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then 1\n when {{first_date}} > {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n > (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then -1\n else 0 end\n\n {%- elif datepart == 'year' -%}\n\n year({{second_date}}) - year({{first_date}})\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set divisor -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n case when {{first_date}} < {{second_date}}\n then ceil((\n {# make sure the timestamps are real, otherwise raise an error asap #}\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n else floor((\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n end\n\n {% if datepart == 'millisecond' %}\n + cast(date_format({{second_date}}, 'SSS') as int)\n - cast(date_format({{first_date}}, 'SSS') as int)\n {% endif %}\n\n {% if datepart == 'microsecond' %}\n {% set capture_str = '[0-9]{4}-[0-9]{2}-[0-9]{2}.[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]{6})' %}\n -- Spark doesn't really support microseconds, so this is a massive hack!\n -- It will only work if the timestamp-string is of the format\n -- 'yyyy-MM-dd-HH mm.ss.SSSSSS'\n + cast(regexp_extract({{second_date}}, '{{capture_str}}', 1) as int)\n - cast(regexp_extract({{first_date}}, '{{capture_str}}', 1) as int)\n {% endif %}\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro datediff not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078332, "supported_languages": null}, "macro.dbt_spark.spark__safe_cast": {"name": "spark__safe_cast", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.spark__safe_cast", "macro_sql": "{% macro spark__safe_cast(field, type) %}\n{%- set field_clean = field.strip('\"').strip(\"'\") if (cast_from_string_unsupported_for(type) and field is string) else field -%}\ncast({{field_clean}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.cast_from_string_unsupported_for"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078588, "supported_languages": null}, "macro.dbt_spark.cast_from_string_unsupported_for": {"name": "cast_from_string_unsupported_for", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.cast_from_string_unsupported_for", "macro_sql": "{% macro cast_from_string_unsupported_for(type) %}\n {{ return(type.lower().startswith('struct') or type.lower().startswith('array') or type.lower().startswith('map')) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0787148, "supported_languages": null}, "macro.dbt_spark.spark__any_value": {"name": "spark__any_value", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_spark.spark__any_value", "macro_sql": "{% macro spark__any_value(expression) -%}\n {#-- return any value (non-deterministic) --#}\n first({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078799, "supported_languages": null}, "macro.dbt_spark.spark__array_concat": {"name": "spark__array_concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt_spark.spark__array_concat", "macro_sql": "{% macro spark__array_concat(array_1, array_2) -%}\n concat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078878, "supported_languages": null}, "macro.dbt_spark.spark__bool_or": {"name": "spark__bool_or", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt_spark.spark__bool_or", "macro_sql": "{% macro spark__bool_or(expression) -%}\n\n max({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078954, "supported_languages": null}, "macro.dbt_spark.spark__split_part": {"name": "spark__split_part", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_spark.spark__split_part", "macro_sql": "{% macro spark__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% if part_number >= 0 %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[({{ part_number - 1 if part_number > 0 else part_number }})]\n\n {% endset %}\n\n {% else %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[(\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1 + {{ part_number }}\n )]\n\n {% endset %}\n\n {% endif %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.079329, "supported_languages": null}, "macro.dbt_spark.spark__array_construct": {"name": "spark__array_construct", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt_spark.spark__array_construct", "macro_sql": "{% macro spark__array_construct(inputs, data_type) -%}\n array( {{ inputs|join(' , ') }} )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.079419, "supported_languages": null}, "macro.dbt_spark.spark__array_append": {"name": "spark__array_append", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt_spark.spark__array_append", "macro_sql": "{% macro spark__array_append(array, new_element) -%}\n {{ array_concat(array, array_construct([new_element])) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.array_concat", "macro.dbt.array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.079514, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0798872, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.079977, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.080045, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0801091, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.080172, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0803041, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.080466, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08062, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.080817, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n {% if config.get(\"dbt_valid_to_current\") %}\n\t{% set source_unique_key = (\"DBT_INTERNAL_DEST.\" ~ columns.dbt_valid_to) | trim %}\n\t{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\tand ({{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null)\n\n {% else %}\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n {% endif %}\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.081193, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0821362, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.082276, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0823748, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.082809, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08289, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.082976, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0835998, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.084287, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085391, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column {{ adapter.quote(column.name) }} {{ column.expanded_data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085542, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08562, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085664, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085738, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085789, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0858781, "supported_languages": null}, "macro.dbt.get_snapshot_table_column_names": {"name": "get_snapshot_table_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_snapshot_table_column_names", "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at', 'dbt_is_deleted': 'dbt_is_deleted'}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085985, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n {% if strategy.hard_deletes == 'new_record' %}\n {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %}\n {% endif %}\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from {{ target_relation }}\n where\n {% if config.get('dbt_valid_to_current') %}\n\t\t{% set source_unique_key = columns.dbt_valid_to | trim %}\n\t\t{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\n\t\t{# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #}\n ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )\n {% else %}\n {{ columns.dbt_valid_to }} is null\n {% endif %}\n\n ),\n\n insertions_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n\n deletes_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n {%- if strategy.hard_deletes == 'new_record' -%}\n ,'False' as {{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from insertions_source_data as source_data\n left outer join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"snapshotted_data\") }}\n or ({{ unique_key_is_not_null(strategy.unique_key, \"snapshotted_data\") }} and (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}\n )\n\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from updates_source_data as source_data\n join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}\n )\n )\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n ,\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n and not (\n --avoid updating the record's valid_to if the latest entry is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n {%- endif %}\n )\n {%- endif %}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n {% set snapshotted_cols = get_list_of_column_names(get_columns_in_relation(target_relation)) %}\n {% set source_sql_cols = get_column_schema_from_query(source_sql) %}\n ,\n deletion_records as (\n\n select\n 'insert' as dbt_change_type,\n {#/*\n If a column has been added to the source it won't yet exist in the\n snapshotted table so we insert a null value as a placeholder for the column.\n */#}\n {%- for col in source_sql_cols -%}\n {%- if col.name in snapshotted_cols -%}\n snapshotted_data.{{ adapter.quote(col.column) }},\n {%- else -%}\n NULL as {{ adapter.quote(col.column) }},\n {%- endif -%}\n {% endfor -%}\n {%- if strategy.unique_key | is_list -%}\n {%- for key in strategy.unique_key -%}\n snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},\n {% endfor -%}\n {%- else -%}\n snapshotted_data.dbt_unique_key as dbt_unique_key,\n {% endif -%}\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},\n {{ new_scd_id }} as {{ columns.dbt_scd_id }},\n 'True' as {{ columns.dbt_is_deleted }}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n and not (\n --avoid inserting a new record if the latest one is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletes\n {%- endif %}\n {%- if strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletion_records\n {%- endif %}\n\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments", "macro.dbt.snapshot_get_time", "macro.dbt.unique_key_fields", "macro.dbt.equals", "macro.dbt.get_dbt_valid_to_current", "macro.dbt.unique_key_join_on", "macro.dbt.unique_key_is_null", "macro.dbt.unique_key_is_not_null", "macro.dbt.get_list_of_column_names", "macro.dbt.get_columns_in_relation", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.088119, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.088233, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , 'False' as {{ columns.dbt_is_deleted }}\n {% endif -%}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.get_dbt_valid_to_current"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.088453, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.088635, "supported_languages": null}, "macro.dbt.get_updated_at_column_data_type": {"name": "get_updated_at_column_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_updated_at_column_data_type", "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08889, "supported_languages": null}, "macro.dbt.check_time_data_types": {"name": "check_time_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.check_time_data_types", "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_updated_at_column_data_type", "macro.dbt.get_snapshot_get_time_data_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.089101, "supported_languages": null}, "macro.dbt.get_dbt_valid_to_current": {"name": "get_dbt_valid_to_current", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_dbt_valid_to_current", "macro_sql": "{% macro get_dbt_valid_to_current(strategy, columns) %}\n {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or \"null\" %}\n coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}})\n as {{ columns.dbt_valid_to }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08924, "supported_languages": null}, "macro.dbt.unique_key_fields": {"name": "unique_key_fields", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_fields", "macro_sql": "{% macro unique_key_fields(unique_key) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n {{ key }} as dbt_unique_key_{{ loop.index }}\n {%- if not loop.last %} , {%- endif %}\n {% endfor %}\n {% else %}\n {{ unique_key }} as dbt_unique_key\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0893881, "supported_languages": null}, "macro.dbt.unique_key_join_on": {"name": "unique_key_join_on", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_join_on", "macro_sql": "{% macro unique_key_join_on(unique_key, identifier, from_identifier) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n\t {% set source_unique_key = (identifier ~ \".dbt_unique_key_\" ~ loop.index) | trim %}\n\t {% set target_unique_key = (from_identifier ~ \".dbt_unique_key_\" ~ loop.index) | trim %}\n\t {{ equals(source_unique_key, target_unique_key) }}\n {%- if not loop.last %} and {%- endif %}\n {% endfor %}\n {% else %}\n {{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.089632, "supported_languages": null}, "macro.dbt.unique_key_is_null": {"name": "unique_key_is_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_is_null", "macro_sql": "{% macro unique_key_is_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is null\n {% else %}\n {{ identifier }}.dbt_unique_key is null\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0897331, "supported_languages": null}, "macro.dbt.unique_key_is_not_null": {"name": "unique_key_is_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_is_not_null", "macro_sql": "{% macro unique_key_is_not_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is not null\n {% else %}\n {{ identifier }}.dbt_unique_key is not null\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.089823, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}\n {% if unique_key | is_list %}\n {% for key in strategy.unique_key %}\n {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }}\n {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }}\n {% endfor %}\n {% endif %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0919418, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n {% set limit = config.get('limit') %}\n {% set sql_header = config.get('sql_header') if flags.REQUIRE_SQL_HEADER_IN_TEST_CONFIGS else none %}\n\n {% set sql_with_limit %}\n {{ get_limit_subquery_sql(sql, limit) }}\n {% endset %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {% if sql_header %}{{ sql_header }}{% endif %}\n {{ get_create_sql(target_relation, sql_with_limit) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {# Since the test failures have already been saved to the database, reuse that result rather than querying again #}\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql_with_limit %}\n\n {% endif %}\n\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {% if sql_header %}{{ sql_header }}{% endif %}\n {# The limit has already been included above, and we do not want to duplicate it again. We also want to be safe for macro overrides treating `limit` as a required parameter. #}\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit=none)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql", "macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.093159, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0934138, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.093545, "supported_languages": null}, "macro.dbt.get_unit_test_sql": {"name": "get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_unit_test_sql", "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_unit_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.093638, "supported_languages": null}, "macro.dbt.default__get_unit_test_sql": {"name": "default__get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_unit_test_sql", "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.093896, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.094045, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.094226, "supported_languages": null}, "macro.dbt.materialization_unit_default": {"name": "materialization_unit_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/unit.sql", "original_file_path": "macros/materializations/tests/unit.sql", "unique_id": "macro.dbt.materialization_unit_default", "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n {% set sql_header = config.get('sql_header') if flags.REQUIRE_SQL_HEADER_IN_TEST_CONFIGS else none %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- set column_name_to_quoted = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- do column_name_to_quoted.update({column.name|lower: column.quoted}) -%}\n {%- endfor -%}\n\n {%- set expected_column_names_quoted = [] -%}\n {%- for column_name in tested_expected_column_names -%}\n {%- do expected_column_names_quoted.append(column_name_to_quoted[column_name|lower]) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types, column_name_to_quoted) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, expected_column_names_quoted) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {% if sql_header %}{{ sql_header }}{% endif %}\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_empty_subquery_sql", "macro.dbt.get_expected_sql", "macro.dbt.get_unit_test_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.095192, "supported_languages": ["sql"]}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0959601, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.096139, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.096252, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0968, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.096912, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.097177, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0981472, "supported_languages": ["sql"]}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n {% do create_indexes(intermediate_relation) %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.099159, "supported_languages": ["sql"]}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0995798, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.099969, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n {{ return(adapter.dispatch('diff_column_data_types', 'dbt')(source_columns, target_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1000772, "supported_languages": null}, "macro.dbt.default__diff_column_data_types": {"name": "default__diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__diff_column_data_types", "macro_sql": "{% macro default__diff_column_data_types(source_columns, target_columns) %}\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.expanded_data_type != tc.expanded_data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.expanded_data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.100428, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.100537, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.100951, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.101467, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set source_unique_key = (\"DBT_INTERNAL_SOURCE.\" ~ unique_key) | trim %}\n\t {% set target_unique_key = (\"DBT_INTERNAL_DEST.\" ~ unique_key) | trim %}\n\t {% set unique_key_match = equals(source_unique_key, target_unique_key) | trim %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns", "macro.dbt.equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.102216, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1023428, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is string %}\n {% set unique_key = [unique_key] %}\n {% endif %}\n\n {%- set unique_key_str = unique_key|join(', ') -%}\n\n delete from {{ target }} as DBT_INTERNAL_DEST\n where ({{ unique_key_str }}) in (\n select distinct {{ unique_key_str }}\n from {{ source }} as DBT_INTERNAL_SOURCE\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.102666, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.102794, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.103076, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.103338, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1037838, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1039, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.10399, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.10412, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104204, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104327, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104411, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104525, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1046069, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104674, "supported_languages": null}, "macro.dbt.get_incremental_microbatch_sql": {"name": "get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_microbatch_sql", "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_microbatch_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104752, "supported_languages": null}, "macro.dbt.default__get_incremental_microbatch_sql": {"name": "default__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_microbatch_sql", "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1048262, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104952, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% set relation_for_indexes = target_relation %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set relation_for_indexes = intermediate_relation %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set relation_for_indexes = temp_relation %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(relation_for_indexes) %}\n {% endif %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1068742, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1075, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n {{ return(adapter.dispatch('check_for_schema_changes', 'dbt')(source_relation, target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__check_for_schema_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.107732, "supported_languages": null}, "macro.dbt.default__check_for_schema_changes": {"name": "default__check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.default__check_for_schema_changes", "macro_sql": "{% macro default__check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.10835, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n {{ return(adapter.dispatch('sync_column_schemas', 'dbt')(on_schema_change, target_relation, schema_changes_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.108466, "supported_languages": null}, "macro.dbt.default__sync_column_schemas": {"name": "default__sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.default__sync_column_schemas", "macro_sql": "{% macro default__sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.108983, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n {{ return(adapter.dispatch('process_schema_changes', 'dbt')(on_schema_change, source_relation, target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__process_schema_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109091, "supported_languages": null}, "macro.dbt.default__process_schema_changes": {"name": "default__process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.default__process_schema_changes", "macro_sql": "{% macro default__process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109462, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109591, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1096509, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109792, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109867, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n {%- set grant_config = config.get('grants') -%}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% if target_relation.database == defer_relation.database and\n target_relation.schema == defer_relation.schema and\n target_relation.identifier == defer_relation.identifier %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n {% endif %}\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.111182, "supported_languages": ["sql"]}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.112628, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1130571, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1134331, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1135402, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.113757, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.113846, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.113909, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.113974, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.114034, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.114114, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.11447, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1146848, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1148071, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.11533, "supported_languages": null}, "macro.dbt.scalar_function_sql": {"name": "scalar_function_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_sql", "macro_sql": "{% macro scalar_function_sql(target_relation) %}\n {{ return(adapter.dispatch('scalar_function_sql', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__scalar_function_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.115777, "supported_languages": null}, "macro.dbt.default__scalar_function_sql": {"name": "default__scalar_function_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_sql", "macro_sql": "{% macro default__scalar_function_sql(target_relation) %}\n {{ scalar_function_create_replace_signature_sql(target_relation) }}\n {{ scalar_function_body_sql() }};\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.scalar_function_create_replace_signature_sql", "macro.dbt.scalar_function_body_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1158478, "supported_languages": null}, "macro.dbt.scalar_function_create_replace_signature_sql": {"name": "scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_create_replace_signature_sql", "macro_sql": "{% macro scalar_function_create_replace_signature_sql(target_relation) %}\n {{ return(adapter.dispatch('scalar_function_create_replace_signature_sql', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__scalar_function_create_replace_signature_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1159308, "supported_languages": null}, "macro.dbt.default__scalar_function_create_replace_signature_sql": {"name": "default__scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_create_replace_signature_sql", "macro_sql": "{% macro default__scalar_function_create_replace_signature_sql(target_relation) %}\n CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}})\n RETURNS {{ model.returns.data_type }}\n {{ scalar_function_volatility_sql() }}\n AS\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.formatted_scalar_function_args_sql", "macro.dbt.scalar_function_volatility_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.11603, "supported_languages": null}, "macro.dbt.formatted_scalar_function_args_sql": {"name": "formatted_scalar_function_args_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.formatted_scalar_function_args_sql", "macro_sql": "{% macro formatted_scalar_function_args_sql() %}\n {{ return(adapter.dispatch('formatted_scalar_function_args_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.116116, "supported_languages": null}, "macro.dbt.default__formatted_scalar_function_args_sql": {"name": "default__formatted_scalar_function_args_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__formatted_scalar_function_args_sql", "macro_sql": "{% macro default__formatted_scalar_function_args_sql() %}\n {% set args = [] %}\n {% for arg in model.arguments -%}\n {%- do args.append(arg.name ~ ' ' ~ arg.data_type) -%}\n {%- endfor %}\n {{ args | join(', ') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1162848, "supported_languages": null}, "macro.dbt.scalar_function_body_sql": {"name": "scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_body_sql", "macro_sql": "{% macro scalar_function_body_sql() %}\n {{ return(adapter.dispatch('scalar_function_body_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__scalar_function_body_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.116382, "supported_languages": null}, "macro.dbt.default__scalar_function_body_sql": {"name": "default__scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_body_sql", "macro_sql": "{% macro default__scalar_function_body_sql() %}\n $$\n {{ model.compiled_code }}\n $$ LANGUAGE SQL\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1164339, "supported_languages": null}, "macro.dbt.scalar_function_volatility_sql": {"name": "scalar_function_volatility_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_volatility_sql", "macro_sql": "{% macro scalar_function_volatility_sql() %}\n {{ return(adapter.dispatch('scalar_function_volatility_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__scalar_function_volatility_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.116509, "supported_languages": null}, "macro.dbt.default__scalar_function_volatility_sql": {"name": "default__scalar_function_volatility_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_volatility_sql", "macro_sql": "{% macro default__scalar_function_volatility_sql() %}\n {% set volatility = model.config.get('volatility') %}\n {% if volatility == 'deterministic' %}\n IMMUTABLE\n {% elif volatility == 'stable' %}\n STABLE\n {% elif volatility == 'non-deterministic' %}\n VOLATILE\n {% elif volatility != none %}\n {# This shouldn't happen unless a new volatility is invented #}\n {% do unsupported_volatility_warning(volatility) %}\n {% endif %}\n {# If no volatility is set, don't add anything and let the data warehouse default it #}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.unsupported_volatility_warning"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.116936, "supported_languages": null}, "macro.dbt.unsupported_volatility_warning": {"name": "unsupported_volatility_warning", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.unsupported_volatility_warning", "macro_sql": "{% macro unsupported_volatility_warning(volatility) %}\n {{ return(adapter.dispatch('unsupported_volatility_warning', 'dbt')(volatility)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__unsupported_volatility_warning"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1170602, "supported_languages": null}, "macro.dbt.default__unsupported_volatility_warning": {"name": "default__unsupported_volatility_warning", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__unsupported_volatility_warning", "macro_sql": "{% macro default__unsupported_volatility_warning(volatility) %}\n {% set msg = \"Found `\" ~ volatility ~ \"` volatility specified on function `\" ~ model.name ~ \"`. This volatility is not supported by \" ~ adapter.type() ~ \", and will be ignored\" %}\n {% do exceptions.warn(msg) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.117184, "supported_languages": null}, "macro.dbt.function_execute_build_sql": {"name": "function_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.function_execute_build_sql", "macro_sql": "{% macro function_execute_build_sql(build_sql, existing_relation, target_relation) %}\n {{ return(adapter.dispatch('function_execute_build_sql', 'dbt')(build_sql, existing_relation, target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__function_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.117384, "supported_languages": null}, "macro.dbt.default__function_execute_build_sql": {"name": "default__function_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.default__function_execute_build_sql", "macro_sql": "{% macro default__function_execute_build_sql(build_sql, existing_relation, target_relation) %}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.117609, "supported_languages": null}, "macro.dbt.get_aggregate_function_create_replace_signature": {"name": "get_aggregate_function_create_replace_signature", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_aggregate_function_create_replace_signature", "macro_sql": "{% macro get_aggregate_function_create_replace_signature(target_relation) %}\n {{ return(adapter.dispatch('get_aggregate_function_create_replace_signature', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_aggregate_function_create_replace_signature"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1179001, "supported_languages": null}, "macro.dbt.default__get_aggregate_function_create_replace_signature": {"name": "default__get_aggregate_function_create_replace_signature", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_aggregate_function_create_replace_signature", "macro_sql": "{% macro default__get_aggregate_function_create_replace_signature(target_relation) %}\n CREATE OR REPLACE AGGREGATE FUNCTION {{ target_relation.render() }} ({{ get_formatted_aggregate_function_args()}})\n RETURNS {{ model.returns.data_type }}\n {{ get_function_language_specifier() }}\n {{ get_aggregate_function_volatility_specifier() }}\n {% if model.get('language') == 'python' %}\n {{ get_function_python_options() }}\n {% endif %}\n AS\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_formatted_aggregate_function_args", "macro.dbt.get_function_language_specifier", "macro.dbt.get_aggregate_function_volatility_specifier", "macro.dbt.get_function_python_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.118081, "supported_languages": null}, "macro.dbt.get_formatted_aggregate_function_args": {"name": "get_formatted_aggregate_function_args", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_formatted_aggregate_function_args", "macro_sql": "{% macro get_formatted_aggregate_function_args() %}\n {{ return(adapter.dispatch('get_formatted_aggregate_function_args', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_formatted_aggregate_function_args"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.118166, "supported_languages": null}, "macro.dbt.default__get_formatted_aggregate_function_args": {"name": "default__get_formatted_aggregate_function_args", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_formatted_aggregate_function_args", "macro_sql": "{% macro default__get_formatted_aggregate_function_args() %}\n {# conveniently we can reuse the sql scalar function args #}\n {{ formatted_scalar_function_args_sql() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.118222, "supported_languages": null}, "macro.dbt.get_function_language_specifier": {"name": "get_function_language_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_function_language_specifier", "macro_sql": "{% macro get_function_language_specifier() %}\n {{ return(adapter.dispatch('get_function_language_specifier', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_function_language_specifier"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.118304, "supported_languages": null}, "macro.dbt.default__get_function_language_specifier": {"name": "default__get_function_language_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_function_language_specifier", "macro_sql": "{% macro default__get_function_language_specifier() %}\n {% set language = model.get('language') %}\n {% if language == 'sql' %}\n {# generally you dont need to specify the language for sql functions #}\n {% elif language == 'python' %}\n LANGUAGE PYTHON\n {% else %}\n {{ 'LANGUAGE ' ~ language.upper() }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.179839, "supported_languages": null}, "macro.dbt.get_aggregate_function_volatility_specifier": {"name": "get_aggregate_function_volatility_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_aggregate_function_volatility_specifier", "macro_sql": "{% macro get_aggregate_function_volatility_specifier() %}\n {{ return(adapter.dispatch('get_aggregate_function_volatility_specifier', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_aggregate_function_volatility_specifier"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.179981, "supported_languages": null}, "macro.dbt.default__get_aggregate_function_volatility_specifier": {"name": "default__get_aggregate_function_volatility_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_aggregate_function_volatility_specifier", "macro_sql": "{% macro default__get_aggregate_function_volatility_specifier() %}\n {{ scalar_function_volatility_sql() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.scalar_function_volatility_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.180064, "supported_languages": null}, "macro.dbt.get_function_python_options": {"name": "get_function_python_options", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_function_python_options", "macro_sql": "{% macro get_function_python_options() %}\n {{ return(adapter.dispatch('get_function_python_options', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_function_python_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.180147, "supported_languages": null}, "macro.dbt.default__get_function_python_options": {"name": "default__get_function_python_options", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_function_python_options", "macro_sql": "{% macro default__get_function_python_options() %}\n RUNTIME_VERSION = '{{ model.config.get('runtime_version') }}'\n HANDLER = '{{ model.config.get('entry_point') }}'\n {% set packages = model.config.get('packages', []) %}\n {% if packages %}\n PACKAGES = ('{{ packages | join(\"','\") }}')\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.18034, "supported_languages": null}, "macro.dbt.materialization_function_default": {"name": "materialization_function_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/function.sql", "original_file_path": "macros/materializations/functions/function.sql", "unique_id": "macro.dbt.materialization_function_default", "macro_sql": "{% materialization function, default, supported_languages=['sql', 'python'] %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.Function) %}\n\n {{ run_hooks(pre_hooks) }}\n\n {% set function_config = this.get_function_config(model) %}\n {% set macro_name = this.get_function_macro_name(function_config) %}\n\n {# Doing this aliasing of adapter.dispatch is a hacky way to disable the static analysis of actually calling adapter.dispatch #}\n {# This is necessary because the static analysis breaks being able to dynamically pass a macro_name #}\n {% set _dispatch = adapter.dispatch %}\n\n {% set build_sql = _dispatch(macro_name, 'dbt')(target_relation) %}\n {{ function_execute_build_sql(build_sql, existing_relation, target_relation) }}\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt.function_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.180772, "supported_languages": ["sql", "python"]}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.180958, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.181129, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.181337, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1814568, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.181586, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1817532, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.181865, "supported_languages": null}, "macro.dbt.get_drop_sql": {"name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182065, "supported_languages": null}, "macro.dbt.default__get_drop_sql": {"name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182234, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182316, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182415, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1825101, "supported_languages": null}, "macro.dbt.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182911, "supported_languages": null}, "macro.dbt.default__get_replace_sql": {"name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation.type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.183627, "supported_languages": null}, "macro.dbt.get_create_intermediate_sql": {"name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.183841, "supported_languages": null}, "macro.dbt.default__get_create_intermediate_sql": {"name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.183991, "supported_languages": null}, "macro.dbt.drop_schema_named": {"name": "drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.drop_schema_named", "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_schema_named"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1841788, "supported_languages": null}, "macro.dbt.default__drop_schema_named": {"name": "default__drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.default__drop_schema_named", "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.184296, "supported_languages": null}, "macro.dbt.get_drop_backup_sql": {"name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1844711, "supported_languages": null}, "macro.dbt.default__get_drop_backup_sql": {"name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.184578, "supported_languages": null}, "macro.dbt.get_rename_sql": {"name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1848068, "supported_languages": null}, "macro.dbt.default__get_rename_sql": {"name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.184999, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1850948, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.185228, "supported_languages": null}, "macro.dbt.get_create_backup_sql": {"name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.185405, "supported_languages": null}, "macro.dbt.default__get_create_backup_sql": {"name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1855412, "supported_languages": null}, "macro.dbt.get_create_sql": {"name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.185751, "supported_languages": null}, "macro.dbt.default__get_create_sql": {"name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.185966, "supported_languages": null}, "macro.dbt.get_rename_intermediate_sql": {"name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186107, "supported_languages": null}, "macro.dbt.default__get_rename_intermediate_sql": {"name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186211, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186338, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186398, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_sql": {"name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186534, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_sql": {"name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186612, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186767, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186836, "supported_languages": null}, "macro.dbt.get_rename_materialized_view_sql": {"name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.18696, "supported_languages": null}, "macro.dbt.default__get_rename_materialized_view_sql": {"name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1871092, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.187374, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1875808, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1877382, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1878111, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.187952, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.188022, "supported_languages": null}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.18837, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.188427, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.188666, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1887438, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.188807, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189343, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1895099, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189655, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189782, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189846, "supported_languages": null}, "macro.dbt.get_replace_table_sql": {"name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189981, "supported_languages": null}, "macro.dbt.default__get_replace_table_sql": {"name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190054, "supported_languages": null}, "macro.dbt.get_rename_table_sql": {"name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_rename_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190172, "supported_languages": null}, "macro.dbt.default__get_rename_table_sql": {"name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190239, "supported_languages": null}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190511, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1907148, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190975, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1914492, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1917288, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.191972, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1921458, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1923451, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.192408, "supported_languages": null}, "macro.dbt.get_replace_view_sql": {"name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.192696, "supported_languages": null}, "macro.dbt.default__get_replace_view_sql": {"name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.192768, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193269, "supported_languages": null}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193362, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1934772, "supported_languages": null}, "macro.dbt.get_rename_view_sql": {"name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_rename_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1936169, "supported_languages": null}, "macro.dbt.default__get_rename_view_sql": {"name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193696, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193876, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193951, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194039, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194255, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194412, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194565, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194686, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1949499, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1956532, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.195957, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1961029, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.196491, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197057, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197427, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197551, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1976728, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197716, "supported_languages": null}, "macro.dbt.get_intervals_between": {"name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197974, "supported_languages": null}, "macro.dbt.default__get_intervals_between": {"name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.198269, "supported_languages": null}, "macro.dbt.date_spine": {"name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.198399, "supported_languages": null}, "macro.dbt.default__date_spine": {"name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.198598, "supported_languages": null}, "macro.dbt.date": {"name": "date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.date", "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__date"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1988358, "supported_languages": null}, "macro.dbt.default__date": {"name": "default__date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.default__date", "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.198976, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199136, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199224, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.19935, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199418, "supported_languages": null}, "macro.dbt.get_powers_of_two": {"name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199641, "supported_languages": null}, "macro.dbt.default__get_powers_of_two": {"name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199845, "supported_languages": null}, "macro.dbt.generate_series": {"name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199947, "supported_languages": null}, "macro.dbt.default__generate_series": {"name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200203, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200331, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2003891, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200618, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200736, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2008681, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200911, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2010481, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2011259, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2012742, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.201345, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.201552, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.201756, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2019022, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2019808, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202115, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202183, "supported_languages": null}, "macro.dbt.equals": {"name": "equals", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/equals.sql", "original_file_path": "macros/utils/equals.sql", "unique_id": "macro.dbt.equals", "macro_sql": "{% macro equals(expr1, expr2) %}\n {{ return(adapter.dispatch('equals', 'dbt') (expr1, expr2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.20233, "supported_languages": null}, "macro.dbt.default__equals": {"name": "default__equals", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/equals.sql", "original_file_path": "macros/utils/equals.sql", "unique_id": "macro.dbt.default__equals", "macro_sql": "{% macro default__equals(expr1, expr2) -%}\n{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %}\n case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null))\n then 0\n else 1\n end = 0\n{%- else -%}\n ({{ expr1 }} = {{ expr2 }})\n{%- endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202465, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202585, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.20266, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202775, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202849, "supported_languages": null}, "macro.dbt.cast": {"name": "cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.cast", "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202975, "supported_languages": null}, "macro.dbt.default__cast": {"name": "default__cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.default__cast", "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2030358, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.203151, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.203209, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2034051, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2034712, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2036622, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2038422, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204191, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204264, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2043588, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204451, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204534, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204611, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204698, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204826, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2049239, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205009, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2051, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2051861, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2052732, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205344, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205499, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205569, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2057018, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205774, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205923, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206113, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206187, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206372, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206465, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206569, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2067, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206762, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.20694, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2070599, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207207, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2072709, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2074199, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207508, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207584, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207673, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2078972, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207967, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208035, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208087, "supported_languages": null}, "macro.dbt.get_snapshot_get_time_data_type": {"name": "get_snapshot_get_time_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.get_snapshot_get_time_data_type", "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.default__snapshot_get_time", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208278, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2083762, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2084138, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208488, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.20863, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208879, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209077, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209198, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209381, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209468, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209532, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209607, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209669, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209991, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210067, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {#-- This ensures microbatch batches get unique temp relations to avoid clobbering --#}\n {% if suffix == '__dbt_tmp' and model.batch %}\n {% set suffix = suffix ~ '_' ~ model.batch.id %}\n {% endif %}\n\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2102349, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.21037, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210477, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210685, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210786, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210868, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210974, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211217, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211325, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211393, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211605, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2118661, "supported_languages": null}, "macro.dbt.collect_freshness_custom_sql": {"name": "collect_freshness_custom_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness_custom_sql", "macro_sql": "{% macro collect_freshness_custom_sql(source, loaded_at_query) %}\n {{ return(adapter.dispatch('collect_freshness_custom_sql', 'dbt')(source, loaded_at_query))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness_custom_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211964, "supported_languages": null}, "macro.dbt.default__collect_freshness_custom_sql": {"name": "default__collect_freshness_custom_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness_custom_sql", "macro_sql": "{% macro default__collect_freshness_custom_sql(source, loaded_at_query) %}\n {% call statement('collect_freshness_custom_sql', fetch_result=True, auto_begin=False) -%}\n with source_query as (\n {{ loaded_at_query }}\n )\n select\n (select * from source_query) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n {% endcall %}\n {{ return(load_result('collect_freshness_custom_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.212112, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.212257, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.212362, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213102, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213182, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2132711, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2133272, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213482, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213568, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213629, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2137332, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.21383, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213931, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.214027, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.214132, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2144508, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.214543, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2146711, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.214787, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.21541, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2156138, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_limit_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2157152, "supported_languages": null}, "macro.dbt.default__get_limit_sql": {"name": "default__get_limit_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_sql", "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ sql }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.215811, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2161121, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216202, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216311, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216398, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216532, "supported_languages": null}, "macro.dbt.validate_doc_columns": {"name": "validate_doc_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.validate_doc_columns", "macro_sql": "{% macro validate_doc_columns(relation, column_dict, existing_column_names) %}\n {% set existing_lower = existing_column_names | map(\"lower\") | list %}\n {% set missing = [] %}\n {% for col_name in column_dict %}\n {% if col_name | lower not in existing_lower %}\n {% do missing.append(col_name) %}\n {% endif %}\n {% endfor %}\n {% if missing | length > 0 %}\n {{ exceptions.warn(\"In relation \" ~ relation.render() ~ \": The following columns are specified in the schema but are not present in the database: \" ~ missing | join(\", \")) }}\n {% endif %}\n {% set filtered = {} %}\n {% for col_name in column_dict if col_name | lower in existing_lower %}\n {% do filtered.update({col_name: column_dict[col_name]}) %}\n {% endfor %}\n {{ return(filtered) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216981, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% set filtered_columns = validate_doc_columns(relation, model.columns, existing_columns) %}\n {% set alter_comment_sql = alter_column_comment(relation, filtered_columns) %}\n {% if alter_comment_sql and alter_comment_sql | trim | length > 0 %}\n {% do run_query(alter_comment_sql) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.validate_doc_columns", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2174249, "supported_languages": null}, "macro.dbt.get_catalog_relations": {"name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2178628, "supported_languages": null}, "macro.dbt.default__get_catalog_relations": {"name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.217987, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218086, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218208, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218299, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2183769, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218462, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2185779, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218684, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218834, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218924, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.219004, "supported_languages": null}, "macro.dbt.get_catalog_for_single_relation": {"name": "get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_for_single_relation", "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_catalog_for_single_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2191591, "supported_languages": null}, "macro.dbt.default__get_catalog_for_single_relation": {"name": "default__get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_for_single_relation", "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2192378, "supported_languages": null}, "macro.dbt.get_relations": {"name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.219313, "supported_languages": null}, "macro.dbt.default__get_relations": {"name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.219388, "supported_languages": null}, "macro.dbt.get_relation_last_modified": {"name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_relation_last_modified"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.21949, "supported_languages": null}, "macro.dbt.default__get_relation_last_modified": {"name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2195961, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220182, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220267, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220414, "supported_languages": null}, "macro.dbt.get_list_of_column_names": {"name": "get_list_of_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_list_of_column_names", "macro_sql": "\n\n{%- macro get_list_of_column_names(columns) -%}\n {% set col_names = [] %}\n {% for col in columns %}\n {% do col_names.append(col.name) %}\n {% endfor %}\n {{ return(col_names) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2205608, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220671, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220783, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220872, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2214458, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2216241, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.22172, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.221895, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.222016, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.222333, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2224672, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.quoted }} {{ column.expanded_data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.quoted }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.222859, "supported_languages": null}, "macro.dbt.get_fixture_sql": {"name": "get_fixture_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_fixture_sql", "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- set column_name_to_quoted = {} -%}\n{%- for column in columns_in_relation -%}\n\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- do column_name_to_quoted.update({column.name|lower: column.quoted}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.safe_cast", "macro.dbt.validate_fixture_rows", "macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.224936, "supported_languages": null}, "macro.dbt.get_expected_sql": {"name": "get_expected_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_expected_sql", "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types, column_name_to_quoted) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.225257, "supported_languages": null}, "macro.dbt.format_row": {"name": "format_row", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.format_row", "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- For string fixture values, strip varchar length to prevent silent truncation (GH-11974) --#}\n {%- if column_value is string and 'varying' in column_type -%}\n {%- set column_type = column_type.split('(')[0] -%}\n {%- endif -%}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.string_literal", "macro.dbt.escape_single_quotes", "macro.dbt.safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.226022, "supported_languages": null}, "macro.dbt.validate_fixture_rows": {"name": "validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.validate_fixture_rows", "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__validate_fixture_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.226166, "supported_languages": null}, "macro.dbt.default__validate_fixture_rows": {"name": "default__validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.default__validate_fixture_rows", "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.226222, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.226644, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2267241, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n\n {#\n We want to get the string of the returned relation by calling .render() in order to skip sample/empty\n mode rendering logic. However, people override the default ref macro, and often return a string instead\n of a relation (like the ref macro does by default). Thus, to make sure we dont blow things up, we have\n to ensure the resolved relation has a .render() method.\n #}\n {%- if resolved.render is defined and resolved.render is callable -%}\n {%- set resolved = resolved.render() -%}\n {%- endif -%}\n\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.22717, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.227363, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\n {# Handle dbt.config.meta_get() calls - use separate dict to avoid overwriting native configs #}\n {%- set meta_dict = {} -%}\n {%- if model.config.meta_keys_used -%}\n {% set meta_dbt_used = zip(model.config.meta_keys_used, model.config.meta_keys_defaults) | list %}\n {%- for key, default in meta_dbt_used -%}\n {%- if model.config.meta and key in model.config.meta -%}\n {%- set value = model.config.meta[key] -%}\n {%- else -%}\n {%- set value = default -%}\n {%- endif -%}\n {%- do meta_dict.update({key: value}) -%}\n {%- endfor -%}\n {%- endif -%}\nconfig_dict = {{ config_dict }}\nmeta_dict = {{ meta_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.227892, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\n @staticmethod\n def meta_get(key, default=None):\n return meta_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.22812, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2281642, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2283971, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.228558, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.228699, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.228839, "supported_languages": null}}, "docs": {"doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {}, "groups": {}, "selectors": {}, "disabled": {}, "parent_map": {"model.dbt_factory.orders_raw": [], "model.dbt_factory.orders_daily": ["model.dbt_factory.orders_raw"], "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91": ["model.dbt_factory.orders_raw"], "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3": ["model.dbt_factory.orders_daily"], "test.dbt_factory.not_null_orders_daily_order_date.e8076bc258": ["model.dbt_factory.orders_daily"]}, "child_map": {"model.dbt_factory.orders_raw": ["model.dbt_factory.orders_daily", "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91"], "model.dbt_factory.orders_daily": ["test.dbt_factory.not_null_orders_daily_order_date.e8076bc258", "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3"], "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91": [], "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3": [], "test.dbt_factory.not_null_orders_daily_order_date.e8076bc258": []}, "group_map": {}, "saved_queries": {}, "semantic_models": {}, "unit_tests": {}, "functions": {}} \ No newline at end of file diff --git a/contrib/dbt_factory/tests/__init__.py b/contrib/dbt_factory/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/tests/conftest.py b/contrib/dbt_factory/tests/conftest.py new file mode 100644 index 00000000..bc24236e --- /dev/null +++ b/contrib/dbt_factory/tests/conftest.py @@ -0,0 +1,39 @@ +import pytest + +from databricks_dbt_factory.dbt_factory import DbtFactory +from databricks_dbt_factory.dbt_task import DbtTaskOptions +from databricks_dbt_factory.task_factory import ( + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, + DbtDependencyResolver, +) + + +def create_dbt_factory(bundle_tests: bool = False) -> DbtFactory: + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + environment_key="Default", + notebook_path="./notebooks/dbt_runner.py", + ) + dbt_options = "--target dev" + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + return DbtFactory(task_factories, bundle_tests=bundle_tests) + + +@pytest.fixture +def dbt_factory() -> DbtFactory: + return create_dbt_factory() + + +@pytest.fixture +def dbt_factory_bundled() -> DbtFactory: + return create_dbt_factory(bundle_tests=True) diff --git a/contrib/dbt_factory/tests/e2e/.gitignore b/contrib/dbt_factory/tests/e2e/.gitignore new file mode 100644 index 00000000..12486301 --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/.gitignore @@ -0,0 +1,4 @@ +target/ +logs/ +dbt_packages/ +.user.yml diff --git a/contrib/dbt_factory/tests/e2e/README.md b/contrib/dbt_factory/tests/e2e/README.md new file mode 100644 index 00000000..e134a3b3 --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/README.md @@ -0,0 +1,58 @@ +# End-to-end test + +`make test-e2e` (from the example root) is the check that a change to the factory won't break real dbt +execution. It generates a fresh project from the [`dbt-factory` template](../../../templates/dbt-factory) +with the factory's default options, points it at **your** Databricks workspace, drops in the fixture +dbt project in this directory, then **deploys the factory-generated job, runs it, verifies the +output, and tears everything down**. Every run destroys its bundle and drops its schema, pass or +fail, so nothing is left behind. + +Unlike `make test` (fast, offline unit tests), this one deploys and runs on a real workspace — so +it's not a CI gate; run it locally before merging a change to the factory. + +## Prerequisites + +- A Databricks CLI profile for your workspace: `databricks auth login --host `. +- A SQL warehouse, and a catalog you can create schemas/tables in. + +## Run it + +``` +DATABRICKS_CONFIG_PROFILE= \ +DBT_FACTORY_HTTP_PATH=/sql/1.0/warehouses/ \ +DBT_FACTORY_CATALOG= \ +make test-e2e +``` + +Optional: `DBT_FACTORY_SCHEMA_PREFIX` (default `dbt_factory_e2e`; a unique `_` schema +is created and dropped each run). + +`make test-e2e` does not read `DBT_FACTORY_SCHEMA` — it injects the per-run `_` +schema into the generated project via `dev_schema`. `DBT_FACTORY_SCHEMA` only applies when you run +`dbt` in this directory directly (see [Parsing the fixture offline](#parsing-the-fixture-offline)). + +## The fixture + +The dbt project here covers every resource type the factory handles — models, a seed, a snapshot, a +`samples.tpch` source, schema tests (including a `severity: warn` test and a cross-model +`relationships` test), and a singular test — so a green run exercises the tricky gating/bundling +paths, not just the happy path. It reads only the ubiquitous `samples` catalog plus its own seed, so +it runs in any workspace. + +## Parsing the fixture offline + +The `dbt_project.yml` and `profiles.yml` in this directory let you run dbt against the fixture on +its own — handy for a quick `dbt parse`/`dbt compile` while editing the models, without generating a +project. `make test-e2e` does not use them (it renders the template and runs dbt inside the generated +project). The profile reads its connection from env vars, all with offline-safe defaults: + +``` +DBT_FACTORY_HTTP_PATH SQL warehouse HTTP path (default: a placeholder; fine for `dbt parse`). +DBT_FACTORY_CATALOG Catalog to read/write (default: main). +DBT_FACTORY_SCHEMA Schema to read/write (default: default). +``` + +``` +cd tests/e2e +uv run dbt parse # reads project files only; no workspace needed +``` diff --git a/contrib/dbt_factory/tests/e2e/dbt_project.yml b/contrib/dbt_factory/tests/e2e/dbt_project.yml new file mode 100644 index 00000000..7d673de5 --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/dbt_project.yml @@ -0,0 +1,10 @@ +name: "dbt_factory_e2e" +version: "1.0.0" +profile: "dbt_factory_e2e" + +model-paths: ["models"] +seed-paths: ["seeds"] +snapshot-paths: ["snapshots"] +test-paths: ["tests"] +target-path: "target" +clean-targets: ["target", "dbt_packages", "logs"] diff --git a/contrib/dbt_factory/tests/e2e/models/customer_orders.sql b/contrib/dbt_factory/tests/e2e/models/customer_orders.sql new file mode 100644 index 00000000..271f23ad --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/models/customer_orders.sql @@ -0,0 +1,11 @@ +-- Downstream of two models: exercises model -> model dependencies. +with customers as (select * from {{ ref('customers') }}), + orders as (select * from {{ ref('orders') }}) +select + customers.customer_id, + customers.customer_name, + count(orders.order_id) as total_orders, + coalesce(sum(orders.amount), 0) as total_amount +from customers +left join orders on customers.customer_id = orders.customer_id +group by customers.customer_id, customers.customer_name diff --git a/contrib/dbt_factory/tests/e2e/models/customers.sql b/contrib/dbt_factory/tests/e2e/models/customers.sql new file mode 100644 index 00000000..65c08c4a --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/models/customers.sql @@ -0,0 +1,6 @@ +-- Base model, no upstream deps. Literal rows keep the e2e fast and workspace-agnostic. +select 1 as customer_id, 'Alice' as customer_name +union all select 2, 'Bob' +union all select 3, 'Charlie' +union all select 4, 'Diana' +union all select 5, 'Eve' diff --git a/contrib/dbt_factory/tests/e2e/models/customers_with_country.sql b/contrib/dbt_factory/tests/e2e/models/customers_with_country.sql new file mode 100644 index 00000000..b63bf27e --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/models/customers_with_country.sql @@ -0,0 +1,10 @@ +-- Downstream of the countries seed: exercises seed -> model gating. +select + c.customer_id, + c.customer_name, + co.country_code, + co.country_name +from {{ ref('customers') }} c +cross join ( + select country_code, country_name from {{ ref('countries') }} where country_code = 'DE' +) co diff --git a/contrib/dbt_factory/tests/e2e/models/orders.sql b/contrib/dbt_factory/tests/e2e/models/orders.sql new file mode 100644 index 00000000..1c826bf6 --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/models/orders.sql @@ -0,0 +1,9 @@ +-- Base model, no upstream deps. +select 1 as order_id, 1 as customer_id, 50.00 as amount +union all select 2, 2, 75.50 +union all select 3, 1, 30.00 +union all select 4, 3, 100.00 +union all select 5, 4, 25.00 +union all select 6, 5, 60.00 +union all select 7, 2, 45.00 +union all select 8, 1, 90.00 diff --git a/contrib/dbt_factory/tests/e2e/models/schema.yml b/contrib/dbt_factory/tests/e2e/models/schema.yml new file mode 100644 index 00000000..fe25a5c8 --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/models/schema.yml @@ -0,0 +1,45 @@ +version: 2 + +sources: + - name: samples_tpch + catalog: samples + schema: tpch + tables: + - name: orders + columns: + - name: o_orderkey + tests: + - not_null + +models: + - name: customers + description: "Base customers." + columns: + - name: customer_id + tests: + - not_null + - unique + - name: orders + description: "Base orders." + columns: + - name: order_id + tests: + - not_null + - name: customer_id + # Cross-model test (references two models) -> emitted as its own task. + tests: + - relationships: + to: ref('customers') + field: customer_id + - name: amount + # warn severity -> the test still runs but does NOT gate downstream tasks. + tests: + - not_null: + config: + severity: warn + - name: customer_orders + description: "Customers joined with their orders (model -> model)." + - name: customers_with_country + description: "Customers joined with the countries seed (seed -> model)." + - name: tpch_orders_summary + description: "Summary over the samples.tpch source (source -> model)." diff --git a/contrib/dbt_factory/tests/e2e/models/tpch_orders_summary.sql b/contrib/dbt_factory/tests/e2e/models/tpch_orders_summary.sql new file mode 100644 index 00000000..3dc64832 --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/models/tpch_orders_summary.sql @@ -0,0 +1,7 @@ +-- Downstream of a Unity Catalog `samples` source: exercises source -> model and source tests. +select + o_orderpriority as order_priority, + count(*) as order_count, + sum(o_totalprice) as total_price +from {{ source('samples_tpch', 'orders') }} +group by o_orderpriority diff --git a/contrib/dbt_factory/tests/e2e/profiles.yml b/contrib/dbt_factory/tests/e2e/profiles.yml new file mode 100644 index 00000000..79ab8c95 --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/profiles.yml @@ -0,0 +1,18 @@ +# Profile for running dbt against the fixture in this directory directly (e.g. `dbt parse` while +# editing the models — see README.md "Parsing the fixture offline"). `make test-e2e` does NOT use +# this file; it renders the template and runs dbt inside the generated project. +# * http_path / catalog / schema come from DBT_FACTORY_* env vars, with offline-safe defaults so +# `dbt parse` runs with no workspace. DBT_FACTORY_SCHEMA defaults to `default`. +# * host / token are injected by the runner notebook at run time (DBT_HOST / DBT_ACCESS_TOKEN). +dbt_factory_e2e: + target: dev + outputs: + dev: + type: databricks + method: http + http_path: "{{ env_var('DBT_FACTORY_HTTP_PATH', '/sql/1.0/warehouses/0000000000000000') }}" + catalog: "{{ env_var('DBT_FACTORY_CATALOG', 'main') }}" + schema: "{{ env_var('DBT_FACTORY_SCHEMA', 'default') }}" + host: "{{ env_var('DBT_HOST', '') }}" + token: "{{ env_var('DBT_ACCESS_TOKEN', '') }}" + threads: 4 diff --git a/contrib/dbt_factory/tests/e2e/run_e2e.py b/contrib/dbt_factory/tests/e2e/run_e2e.py new file mode 100644 index 00000000..ceeb560a --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/run_e2e.py @@ -0,0 +1,258 @@ +""" +End-to-end test for the dbt-factory example. + +It generates a fresh project from the ``dbt-factory`` template with the factory's default options, +points it at YOUR workspace, drops in the fixture dbt project next to this file, then **deploys the +factory-generated job, runs it, and verifies it succeeded** — before tearing everything down. This +is the check that a change to the factory won't break real dbt execution. + +Run it with ``make test-e2e`` from the example root. Required environment: + + DATABRICKS_CONFIG_PROFILE CLI profile for your workspace (auth + host). + DBT_FACTORY_HTTP_PATH SQL warehouse HTTP path, e.g. /sql/1.0/warehouses/. + DBT_FACTORY_CATALOG Catalog to create the throwaway schema + tables in (write access). + +Optional: + DBT_FACTORY_SCHEMA_PREFIX Schema-name prefix (default: dbt_factory_e2e). A unique + _ schema is created and dropped per run. + +Nothing is left behind: the run destroys its bundle and drops its schema, pass or fail. +""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +FIXTURE_DIR = Path(__file__).resolve().parent +REPO_ROOT = FIXTURE_DIR.parents[3] +TEMPLATE_DIR = REPO_ROOT / "contrib" / "templates" / "dbt-factory" + +PROJECT_NAME = "dbt_factory_e2e" + +# Tables the fixture materializes, with the minimum row count that proves they actually ran. +EXPECTED_TABLES = { + "customers": 5, + "orders": 8, + "customer_orders": 5, + "customers_with_country": 5, + "tpch_orders_summary": 1, + "orders_snapshot": 8, +} + + +class Config: + def __init__(self) -> None: + self.profile = _require("DATABRICKS_CONFIG_PROFILE") + self.http_path = _require("DBT_FACTORY_HTTP_PATH") + self.catalog = _require("DBT_FACTORY_CATALOG") + self.warehouse_id = self.http_path.rstrip("/").split("/")[-1] + self.prefix = os.environ.get("DBT_FACTORY_SCHEMA_PREFIX", "dbt_factory_e2e") + + +def _require(name: str) -> str: + value = os.environ.get(name) + if not value: + sys.exit(f"ERROR: environment variable {name} is required (see the module docstring).") + return value + + +def _run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess: + print(f" $ {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, text=True) + if check and result.returncode != 0: + raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") + return result + + +SQL_DEADLINE_SECONDS = 600 + + +def _api(method: str, path: str, body: dict | None = None) -> dict: + cmd = ["databricks", "api", method, path] + if body is not None: + cmd += ["--json", json.dumps(body)] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"API call failed ({method} {path}): {result.stderr.strip()}") + return json.loads(result.stdout) if result.stdout.strip() else {} + + +def _sql(cfg: Config, statement: str, schema: str | None = None) -> list[list]: + """Runs a SQL statement on the warehouse via the Statement Execution API; returns result rows. + The API caps its synchronous wait at 50s, which a cold warehouse can exceed, so this polls the + statement until it reaches a terminal state instead of cancelling it.""" + body = { + "warehouse_id": cfg.warehouse_id, + "statement": statement, + "catalog": cfg.catalog, + "wait_timeout": "50s", + "on_wait_timeout": "CONTINUE", + } + if schema: + body["schema"] = schema + payload = _api("post", "/api/2.0/sql/statements", body) + deadline = time.time() + SQL_DEADLINE_SECONDS + while payload.get("status", {}).get("state") in ("PENDING", "RUNNING"): + if time.time() > deadline: + raise RuntimeError(f"SQL not finished after {SQL_DEADLINE_SECONDS}s: {statement}") + time.sleep(5) + payload = _api("get", f"/api/2.0/sql/statements/{payload['statement_id']}") + state = payload.get("status", {}).get("state") + if state != "SUCCEEDED": + raise RuntimeError(f"SQL did not succeed ({state}): {statement}\n{json.dumps(payload.get('status', {}))}") + return payload.get("result", {}).get("data_array", []) or [] + + +def _init_project(cfg: Config, out_dir: Path, schema: str) -> Path: + # Set only the values that point the project at this workspace. Everything else (test bundling, + # environment key, extra dbt options) is left at the template's defaults — the factory's + # out-of-the-box behavior, which is exactly what we want the e2e to exercise. + config_file = out_dir / "init-config.json" + config_file.write_text( + json.dumps( + { + "project_name": PROJECT_NAME, + "default_catalog": cfg.catalog, + "dev_schema": schema, + "http_path": cfg.http_path, + } + ) + ) + # Run `bundle init` from the neutral temp dir, NOT from inside the example bundle — otherwise the + # CLI resolves auth against the example's placeholder host and refuses. + _run( + [ + "databricks", + "bundle", + "init", + str(TEMPLATE_DIR), + "--config-file", + str(config_file), + "--output-dir", + str(out_dir), + ], + cwd=out_dir, + ) + return out_dir / PROJECT_NAME + + +def _fill_fixture(project: Path) -> None: + """Replace the starter models with the richer fixture dbt project next to this script.""" + src = project / "src" + shutil.rmtree(src / "models" / "example", ignore_errors=True) + for sub in ("models", "seeds", "snapshots", "tests"): + for f in sorted((FIXTURE_DIR / sub).iterdir()): + if f.is_file(): + shutil.copy2(f, src / sub / f.name) + + +def _assert_parse_cache_synced(project: Path) -> None: + """Guard the parse-cache optimization: the git-ignored `target/partial_parse.msgpack` must reach + the workspace (via `sync.include`) or every task silently full-parses instead of injecting it. + A missing file here is the regression that the old `git add -f` opt-in caused. Tooling hiccups + (summary/list) only warn; a file that is definitively absent fails the run.""" + summary = subprocess.run( + ["databricks", "bundle", "summary", "--target", "dev", "-o", "json"], + cwd=project, + capture_output=True, + text=True, + ) + if summary.returncode != 0: + print(f" WARNING: bundle summary failed, skipping parse-cache check: {summary.stderr.strip()}") + return + try: + file_path = json.loads(summary.stdout)["workspace"]["file_path"] + except (json.JSONDecodeError, KeyError) as exc: + print(f" WARNING: could not read workspace.file_path, skipping parse-cache check: {exc}") + return + listing = subprocess.run( + ["databricks", "workspace", "list", f"{file_path}/target", "-o", "json"], + cwd=project, # resolve auth against the rendered project's host, not the example bundle's + capture_output=True, + text=True, + ) + if listing.returncode != 0: + print(f" WARNING: could not list {file_path}/target, skipping parse-cache check: {listing.stderr.strip()}") + return + names = { + o.get("path", "").rsplit("/", 1)[-1] for o in (json.loads(listing.stdout) if listing.stdout.strip() else []) + } + if "partial_parse.msgpack" not in names: + raise RuntimeError(f"partial_parse.msgpack not synced to {file_path}/target (found: {sorted(names)})") + print(f" parse cache synced: {file_path}/target/partial_parse.msgpack") + + +def _verify_output(cfg: Config, schema: str) -> list[str]: + """Query each expected table's row count; return a list of failure messages (empty == all good).""" + failures = [] + for table, minimum in EXPECTED_TABLES.items(): + rows = _sql(cfg, f"SELECT count(*) FROM {cfg.catalog}.{schema}.{table}", schema=schema) + count = int(rows[0][0]) if rows and rows[0] else 0 + status = "ok" if count >= minimum else "TOO FEW" + print(f" {table:26} rows={count:<6} (expected >= {minimum}) {status}") + if count < minimum: + failures.append(f"{table}: {count} rows (< {minimum})") + return failures + + +def run(cfg: Config) -> bool: + schema = f"{cfg.prefix}_{time.strftime('%Y%m%d_%H%M%S')}" + job = f"{PROJECT_NAME}_job" + work = Path(tempfile.mkdtemp(prefix="dbtfactory_e2e_")) + project = None + print(f"\n===== dbt-factory e2e — schema {cfg.catalog}.{schema} =====") + try: + print(" [1/6] create throwaway schema") + _sql(cfg, f"CREATE SCHEMA IF NOT EXISTS {cfg.catalog}.{schema}") + print(" [2/6] init project from template + drop in fixture") + project = _init_project(cfg, work, schema) + _fill_fixture(project) + print(" [3/6] install deps + generate dbt manifest") + _run(["uv", "sync", "--dev"], cwd=project) + _run(["uv", "run", "dbt", "parse", "--profiles-dir", "dbt_profiles"], cwd=project) + print(" [4/6] deploy the factory-generated job") + _run(["databricks", "bundle", "deploy", "--target", "dev"], cwd=project) + _assert_parse_cache_synced(project) + print(" [5/6] run the job (blocks; non-zero exit if any task fails)") + run_result = _run(["databricks", "bundle", "run", job, "--target", "dev"], cwd=project, check=False) + job_ok = run_result.returncode == 0 + print(f" job result: {'SUCCESS' if job_ok else 'FAILED'}") + print(" [6/6] verify output tables were materialized") + failures = _verify_output(cfg, schema) if job_ok else ["job did not succeed; skipped table checks"] + for f in failures: + print(f" FAIL: {f}") + return job_ok and not failures + except Exception as exc: # noqa: BLE001 - surface any failure + print(f" ERROR: {exc}") + return False + finally: + print(" cleanup: destroy bundle + drop schema") + if project and project.exists(): + _run(["databricks", "bundle", "destroy", "--target", "dev", "--auto-approve"], cwd=project, check=False) + for attempt in (1, 2): + try: + _sql(cfg, f"DROP SCHEMA IF EXISTS {cfg.catalog}.{schema} CASCADE") + break + except Exception as exc: # noqa: BLE001 + if attempt == 2: + print(f" WARNING: could not drop schema {schema}: {exc}") + else: + print(f" schema drop failed, retrying: {exc}") + time.sleep(10) + shutil.rmtree(work, ignore_errors=True) + + +def main() -> None: + passed = run(Config()) + print(f"\n===== e2e {'PASS' if passed else 'FAIL'} =====") + if not passed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/contrib/dbt_factory/tests/e2e/seeds/countries.csv b/contrib/dbt_factory/tests/e2e/seeds/countries.csv new file mode 100644 index 00000000..7f4a79af --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/seeds/countries.csv @@ -0,0 +1,6 @@ +country_code,country_name +DE,Germany +US,United States +FR,France +UK,United Kingdom +JP,Japan diff --git a/contrib/dbt_factory/tests/e2e/seeds/schema.yml b/contrib/dbt_factory/tests/e2e/seeds/schema.yml new file mode 100644 index 00000000..b643f1d5 --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/seeds/schema.yml @@ -0,0 +1,13 @@ +version: 2 + +seeds: + - name: countries + description: "Country code lookup seed." + columns: + - name: country_code + tests: + - not_null + - unique + - name: country_name + tests: + - not_null diff --git a/contrib/dbt_factory/tests/e2e/snapshots/orders_snapshot.sql b/contrib/dbt_factory/tests/e2e/snapshots/orders_snapshot.sql new file mode 100644 index 00000000..317f478b --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/snapshots/orders_snapshot.sql @@ -0,0 +1,11 @@ +{% snapshot orders_snapshot %} +{{ + config( + target_schema=target.schema, + unique_key='order_id', + strategy='check', + check_cols=['amount'] + ) +}} +select * from {{ ref('orders') }} +{% endsnapshot %} diff --git a/contrib/dbt_factory/tests/e2e/snapshots/schema.yml b/contrib/dbt_factory/tests/e2e/snapshots/schema.yml new file mode 100644 index 00000000..0cf988ef --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/snapshots/schema.yml @@ -0,0 +1,9 @@ +version: 2 + +snapshots: + - name: orders_snapshot + description: "SCD2 snapshot of orders (tracks amount changes)." + columns: + - name: order_id + tests: + - not_null diff --git a/contrib/dbt_factory/tests/e2e/tests/test_no_negative_amounts.sql b/contrib/dbt_factory/tests/e2e/tests/test_no_negative_amounts.sql new file mode 100644 index 00000000..c5b3e738 --- /dev/null +++ b/contrib/dbt_factory/tests/e2e/tests/test_no_negative_amounts.sql @@ -0,0 +1,4 @@ +-- Singular test: fails if any order has a negative amount. +select order_id, amount +from {{ ref('orders') }} +where amount < 0 diff --git a/contrib/dbt_factory/tests/test_data/expected_tasks.json b/contrib/dbt_factory/tests/test_data/expected_tasks.json new file mode 100644 index 00000000..c8a56f3d --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/expected_tasks.json @@ -0,0 +1,567 @@ +[ + { + "task_key": "diamonds_list_colors_model", + "depends_on": [ + { + "task_key": "diamonds_four_cs_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.diamonds_list_colors,package:dbt_demo,file:diamonds_list_colors.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "diamonds_prices_model", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.diamonds_prices,package:dbt_demo,file:diamonds_prices.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "diamonds_four_cs_model", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.diamonds_four_cs,package:dbt_demo,file:diamonds_four_cs.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "zzz_game_details_model", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.zzz_game_details,package:dbt_demo,file:zzz_game_details.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "second_dbt_model_model", + "depends_on": [ + { + "task_key": "first_dbt_model_model" + }, + { + "task_key": "not_null_first_dbt_model_id_test" + }, + { + "task_key": "unique_first_dbt_model_id_test" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model2.second_dbt_model,package:dbt_demo,file:second_dbt_model.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "first_dbt_model_model", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model2.first_dbt_model,package:dbt_demo,file:first_dbt_model.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "zzz_game_details_check_scores_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.data_tests.zzz_game_details_check_scores,package:dbt_demo,file:zzz_game_details_check_scores.sql,resource_type:test --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "zzz_game_details_check_dates_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.data_tests.zzz_game_details_check_dates,package:dbt_demo,file:zzz_game_details_check_dates.sql,resource_type:test --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "zzz_win_loss_records_check_records_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.data_tests.zzz_win_loss_records_check_records,package:dbt_demo,file:zzz_win_loss_records_check_records.sql,resource_type:test --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "diamonds_list_colors_test_test", + "depends_on": [ + { + "task_key": "diamonds_list_colors_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.data_tests.diamonds_list_colors_test,package:dbt_demo,file:diamonds_list_colors_test.sql,resource_type:test --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "country_codes_seed", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt seed --select fqn:dbt_demo.country_codes,package:dbt_demo,file:country_codes.csv,resource_type:seed --target dev\"]" + } + } + }, + { + "task_key": "unique_zzz_game_details_game_id_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.unique_zzz_game_details_game_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:unique --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_game_id_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_game_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_home_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_home,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:accepted_values --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_visitor_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_visitor,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:accepted_values --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_home_score_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_home_score,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_visitor_score_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_visitor_score,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_winner_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_winner,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:accepted_values --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_date_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_date,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "unique_first_dbt_model_id_test", + "depends_on": [ + { + "task_key": "first_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.unique_first_dbt_model_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:unique --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_first_dbt_model_id_test", + "depends_on": [ + { + "task_key": "first_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.not_null_first_dbt_model_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "unique_second_dbt_model_id_test", + "depends_on": [ + { + "task_key": "second_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.unique_second_dbt_model_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:unique --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_second_dbt_model_id_test", + "depends_on": [ + { + "task_key": "second_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.not_null_second_dbt_model_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_second_dbt_model_amount_usd_test", + "depends_on": [ + { + "task_key": "second_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.not_null_second_dbt_model_amount_usd,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "changed_country_model", + "depends_on": [ + { + "task_key": "country_snapshot_snapshot" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.changed_country,package:dbt_demo,file:changed_country.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "country_model", + "depends_on": [ + { + "task_key": "country_codes_seed" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.country,package:dbt_demo,file:country.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "country_snapshot_snapshot", + "depends_on": [ + { + "task_key": "country_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt snapshot --select fqn:dbt_demo.country_snapshot.country_snapshot,package:dbt_demo,file:country_snapshot.sql,resource_type:snapshot --target dev\"]" + } + } + }, + { + "task_key": "zzz_win_loss_records_model", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + }, + { + "task_key": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_test" + }, + { + "task_key": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_test" + }, + { + "task_key": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_test" + }, + { + "task_key": "not_null_zzz_game_details_date_test" + }, + { + "task_key": "not_null_zzz_game_details_game_id_test" + }, + { + "task_key": "not_null_zzz_game_details_home_score_test" + }, + { + "task_key": "not_null_zzz_game_details_home_test" + }, + { + "task_key": "not_null_zzz_game_details_visitor_score_test" + }, + { + "task_key": "not_null_zzz_game_details_visitor_test" + }, + { + "task_key": "not_null_zzz_game_details_winner_test" + }, + { + "task_key": "unique_zzz_game_details_game_id_test" + }, + { + "task_key": "zzz_game_details_check_dates_test" + }, + { + "task_key": "zzz_game_details_check_scores_test" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.zzz_win_loss_records,package:dbt_demo,file:zzz_win_loss_records.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "unique_zzz_win_loss_records_team_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.unique_zzz_win_loss_records_team,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:unique --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_win_loss_records_team_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_win_loss_records_team,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_win_loss_records_wins_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_win_loss_records_wins,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_win_loss_records_losses_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_win_loss_records_losses,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "unit_test_dbt_demo_diamonds_list_colors_test_get_distinct_colors", + "depends_on": [ + { + "task_key": "diamonds_list_colors_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.diamonds_list_colors.test_get_distinct_colors,package:dbt_demo,file:test_get_distinct_colors.yml,resource_type:unit_test --target dev --indirect-selection empty\"]" + } + } + } +] diff --git a/contrib/dbt_factory/tests/test_data/manifest.json b/contrib/dbt_factory/tests/test_data/manifest.json new file mode 100644 index 00000000..44282d07 --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/manifest.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": "1.8.7", "generated_at": "2024-12-06T13:55:35.349350Z", "invocation_id": "a2d9eb4c-6787-4ede-b5ce-44e61f3769b2", "env": {}, "project_name": "dbt_demo", "project_id": "f005e70093f1ddd3daa6316f5ec6f69e", "user_id": "b802ff80-7d9c-4428-8a39-5f56cf164bf8", "send_anonymous_usage_stats": true, "adapter_type": "databricks"}, "nodes": {"model.dbt_demo.diamonds_list_colors": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_list_colors", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_list_colors.sql", "original_file_path": "src/models/sql_model1/diamonds_list_colors.sql", "unique_id": "model.dbt_demo.diamonds_list_colors", "fqn": ["dbt_demo", "sql_model1", "diamonds_list_colors"], "alias": "diamonds_list_colors", "checksum": {"name": "sha256", "checksum": "fb8e076fbc40ae30c2441fbc660922d1d3428cf1557cc8f400dbd69caa78c266"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.194898, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_list_colors`", "raw_code": "-- create a view\nselect distinct color\nfrom {{ ref('diamonds_four_cs') }}\nsort by color asc", "language": "sql", "refs": [{"name": "diamonds_four_cs", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_four_cs"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "compiled": true, "compiled_code": "-- create a view\nselect distinct color\nfrom `dbt_demo`.`dbt_dev`.`diamonds_four_cs`\nsort by color asc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_prices": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_prices", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_prices.sql", "original_file_path": "src/models/sql_model1/diamonds_prices.sql", "unique_id": "model.dbt_demo.diamonds_prices", "fqn": ["dbt_demo", "sql_model1", "diamonds_prices"], "alias": "diamonds_prices", "checksum": {"name": "sha256", "checksum": "809b1318095c2871ab3e9d98af3cc11233e1fbb791e19682fbb38070074e53d2"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.207654, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_prices`", "raw_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum({{ cents_to_dollars('usage_quantity') }}) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "compiled": true, "compiled_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum(\n (usage_quantity / 100)::numeric(16, 2)\n) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_four_cs": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_four_cs", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_four_cs.sql", "original_file_path": "src/models/sql_model1/diamonds_four_cs.sql", "unique_id": "model.dbt_demo.diamonds_four_cs", "fqn": ["dbt_demo", "sql_model1", "diamonds_four_cs"], "alias": "diamonds_four_cs", "checksum": {"name": "sha256", "checksum": "85a0d2493625371fdfbd2d8261dcc22335301c88e08791638ad760e6cb7dd614"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.264364, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_four_cs`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom {{ source('raw', 'diamonds') }} -- equivalent of: from hive_metastore.default.diamonds", "language": "sql", "refs": [], "sources": [["raw", "diamonds"]], "metrics": [], "depends_on": {"macros": [], "nodes": ["source.dbt_demo.raw.diamonds"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom `hive_metastore`.`default`.`diamonds` -- equivalent of: from hive_metastore.default.diamonds", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.zzz_game_details": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_game_details", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_game_details.sql", "original_file_path": "src/models/sql_model1/zzz_game_details.sql", "unique_id": "model.dbt_demo.zzz_game_details", "fqn": ["dbt_demo", "sql_model1", "zzz_game_details"], "alias": "zzz_game_details", "checksum": {"name": "sha256", "checksum": "07a0ddce4e171f5838f27124b3574a77cbab23fac66bba7a1b8f87f6bfbd653f"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {"game_id": {"name": "game_id", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home": {"name": "home", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor": {"name": "visitor", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home_score": {"name": "home_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor_score": {"name": "visitor_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "winner": {"name": "winner", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "date": {"name": "date", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.43343, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_game_details`", "raw_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.{{ target.name }}_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "compiled": true, "compiled_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.dev_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.second_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "second_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/second_dbt_model.sql", "original_file_path": "src/models/sql_model2/second_dbt_model.sql", "unique_id": "model.dbt_demo.second_dbt_model", "fqn": ["dbt_demo", "sql_model2", "second_dbt_model"], "alias": "second_dbt_model", "checksum": {"name": "sha256", "checksum": "226c0e1d47e6f9bee7430c7acffadc0d239fa185558a2de703c5f0ab0b01d218"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}, "amount_usd": {"name": "amount_usd", "description": "The primary key for this table", "meta": {}, "data_type": "decimal", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}}, "created_at": 1733486794.536133, "relation_name": "`dbt_demo`.`dbt_dev`.`second_dbt_model`", "raw_code": "-- Use the `ref` function to select from other models\n\nselect\n *, {{ cents_to_dollars('id') }} as amount_usd\nfrom {{ ref('first_dbt_model') }}\nwhere id = 1", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "compiled": true, "compiled_code": "-- Use the `ref` function to select from other models\n\nselect\n *, \n (id / 100)::numeric(16, 2)\n as amount_usd\nfrom `dbt_demo`.`dbt_dev`.`first_dbt_model`\nwhere id = 1", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "d4b533efdb37067787094e25d1860ddcc473cb5cf92ebe2c358011ff2866658f"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.first_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "first_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/first_dbt_model.sql", "original_file_path": "src/models/sql_model2/first_dbt_model.sql", "unique_id": "model.dbt_demo.first_dbt_model", "fqn": ["dbt_demo", "sql_model2", "first_dbt_model"], "alias": "first_dbt_model", "checksum": {"name": "sha256", "checksum": "91d9a8b1e5c899463e52c50492b7b3c7df99fa6635c868f9165ca83e5f3b13f7"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}, "materialized": "table"}, "created_at": 1733486794.534828, "relation_name": "`dbt_demo`.`dbt_dev`.`first_dbt_model`", "raw_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n{{ config(materialized='table') }}\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "compiled": true, "compiled_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "a31b0e38eb3a9f0c0ed2f45bd615732c39e48b7f698f50a36748a687fd6d7289"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.zzz_game_details_check_scores": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_scores", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_scores.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_scores.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_scores", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_scores"], "alias": "zzz_game_details_check_scores", "checksum": {"name": "sha256", "checksum": "49cca9739f2738a4609833d267ade1a137756523f0659ec02f0bd31adb3c5789"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.31489, "relation_name": null, "raw_code": "-- This sport allows no negative scores or tie games.\n-- For this test to pass, this query must return no results.\n\nselect home_score, visitor_score\nfrom {{ ref('zzz_game_details') }}\nwhere home_score < 0\nor visitor_score < 0\nor home_score = visitor_score", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_game_details_check_dates": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_dates", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_dates.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_dates.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_dates", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_dates"], "alias": "zzz_game_details_check_dates", "checksum": {"name": "sha256", "checksum": "ac2ae3e40e5b734327e8a60a8c944e535c472314feee78d9320da1f7ed515918"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3213592, "relation_name": null, "raw_code": "-- This season's games happened between 2020-12-12 and 2021-02-06.\n-- For this test to pass, this query must return no results.\n\nselect date\nfrom {{ ref('zzz_game_details') }}\nwhere date < '2020-12-12'\nor date > '2021-02-06'", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_win_loss_records_check_records": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_win_loss_records_check_records", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_win_loss_records_check_records.sql", "original_file_path": "tests/data_tests/zzz_win_loss_records_check_records.sql", "unique_id": "test.dbt_demo.zzz_win_loss_records_check_records", "fqn": ["dbt_demo", "data_tests", "zzz_win_loss_records_check_records"], "alias": "zzz_win_loss_records_check_records", "checksum": {"name": "sha256", "checksum": "8840acce153171ad015abfe10626d1220cc9533e98c859a2b483f0d25d2e4903"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.323633, "relation_name": null, "raw_code": "-- Each team participated in 4 games this season.\n-- For this test to pass, this query must return no results.\n\nselect wins, losses\nfrom {{ ref('zzz_win_loss_records') }}\nwhere wins < 0 or wins > 4\nor losses < 0 or losses > 4\nor (wins + losses) > 4", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.diamonds_list_colors_test": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "diamonds_list_colors_test", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/diamonds_list_colors_test.sql", "original_file_path": "tests/data_tests/diamonds_list_colors_test.sql", "unique_id": "test.dbt_demo.diamonds_list_colors_test", "fqn": ["dbt_demo", "data_tests", "diamonds_list_colors_test"], "alias": "diamonds_list_colors_test", "checksum": {"name": "sha256", "checksum": "4a4d3ee2fa6aad9a7fc4b248111aa6432b67374e217793fbf29d92c2504f13be"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3264809, "relation_name": null, "raw_code": "SELECT color, COUNT(*)\nFROM {{ ref('diamonds_list_colors') }}\nGROUP BY color\nHAVING COUNT(*) > 1", "language": "sql", "refs": [{"name": "diamonds_list_colors", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "seed.dbt_demo.country_codes": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_codes", "resource_type": "seed", "package_name": "dbt_demo", "path": "country_codes.csv", "original_file_path": "src/seeds/country_codes.csv", "unique_id": "seed.dbt_demo.country_codes", "fqn": ["dbt_demo", "country_codes"], "alias": "country_codes", "checksum": {"name": "sha256", "checksum": "b54c38e4337b053b07cdc37fa5364e8f159b4e37b77d36281a8d2d11a2c09b8c"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "delimiter": ",", "quote_columns": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.388313, "relation_name": "`dbt_demo`.`dbt_dev`.`country_codes`", "raw_code": "", "root_path": "/Users/marcin.wojtyczka/Documents/repos/dbt_demo", "depends_on": {"macros": []}}, "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_game_details_game_id"], "alias": "unique_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.462498, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "unique", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_game_id"], "alias": "not_null_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4634, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home"], "alias": "not_null_zzz_game_details_home", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.464213, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3"}, "created_at": 1733486794.4649322, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor"], "alias": "not_null_zzz_game_details_visitor", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.474986, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272"}, "created_at": 1733486794.4757178, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home_score"], "alias": "not_null_zzz_game_details_home_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4776351, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor_score"], "alias": "not_null_zzz_game_details_visitor_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4783318, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_winner", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_winner.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_winner"], "alias": "not_null_zzz_game_details_winner", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.479018, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca"}, "created_at": 1733486794.4797802, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_date", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_date.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_date"], "alias": "not_null_zzz_game_details_date", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4815848, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "date", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "date", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_first_dbt_model_id.9dda05743e", "fqn": ["dbt_demo", "models", "sql_model2", "unique_first_dbt_model_id"], "alias": "unique_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.536581, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_first_dbt_model_id"], "alias": "not_null_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537291, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4", "fqn": ["dbt_demo", "models", "sql_model2", "unique_second_dbt_model_id"], "alias": "unique_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537977, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_id"], "alias": "not_null_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.538719, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_amount_usd", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_amount_usd.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_amount_usd"], "alias": "not_null_second_dbt_model_amount_usd", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.539395, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "amount_usd", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "amount_usd", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "model.dbt_demo.changed_country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "changed_country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/changed_country.sql", "original_file_path": "src/models/sql_model1/changed_country.sql", "unique_id": "model.dbt_demo.changed_country", "fqn": ["dbt_demo", "sql_model1", "changed_country"], "alias": "changed_country", "checksum": {"name": "sha256", "checksum": "e1f9599d0e633856772d321bfc75b89ead550df1ba8fe3dc40a8102c9cc2eb9d"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/changed_country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733490496.062186, "relation_name": "`dbt_demo`.`dbt_dev`.`changed_country`", "raw_code": "{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect * from {{ ref('country_snapshot') }}", "language": "sql", "refs": [{"name": "country_snapshot", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["snapshot.dbt_demo.country_snapshot"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/changed_country.sql", "compiled": true, "compiled_code": "\nselect * from `dbt_demo`.`dbt_dev`.`country_snapshot`", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/country.sql", "original_file_path": "src/models/sql_model1/country.sql", "unique_id": "model.dbt_demo.country", "fqn": ["dbt_demo", "sql_model1", "country"], "alias": "country", "checksum": {"name": "sha256", "checksum": "be347ee69b8c27dec884b8bd692e1aa0ffdac4fb2578c410d048dc9ed16b0b08"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733492304.094096, "relation_name": "`dbt_demo`.`dbt_dev`.`country`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n country_code,\n country_name\nfrom {{ ref('country_codes') }} -- use seed data to populate the table", "language": "sql", "refs": [{"name": "country_codes", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["seed.dbt_demo.country_codes"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/country.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n country_code,\n country_name\nfrom `dbt_demo`.`dbt_dev`.`country_codes` -- use seed data to populate the table", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "snapshot.dbt_demo.country_snapshot": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_snapshot", "resource_type": "snapshot", "package_name": "dbt_demo", "path": "country_snapshot.sql", "original_file_path": "src/snapshots/country_snapshot.sql", "unique_id": "snapshot.dbt_demo.country_snapshot", "fqn": ["dbt_demo", "country_snapshot", "country_snapshot"], "alias": "country_snapshot", "checksum": {"name": "sha256", "checksum": "100b8c3c320732e98704fe520db0c0bcbf9f1fccf565fccd5807b87cd0d01984"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "country_code", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "strategy": "check", "target_schema": "dbt_dev", "target_database": "dbt_demo", "updated_at": null, "check_cols": ["country_code"]}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"target_database": "dbt_demo", "target_schema": "dbt_dev", "unique_key": "country_code", "strategy": "check", "check_cols": ["country_code"]}, "created_at": 1733493205.419168, "relation_name": "`dbt_demo`.`dbt_dev`.`country_snapshot`", "raw_code": "\n\n{% set unique_key = 'country_code' %}\n{% set strategy = 'check' %}\n{% set check_cols = ['country_code'] %}\n{% set target_database = 'dbt_demo' %}\n{% set target_schema = 'dbt_dev' %}\n\n{{\n config(\n target_database=target_database,\n target_schema=target_schema,\n unique_key=unique_key,\n strategy=strategy,\n check_cols=check_cols\n )\n}}\n\n-- SQL query for the snapshot\nSELECT *\nFROM {{ ref('country') }}\n\n", "language": "sql", "refs": [{"name": "country", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.country"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "model.dbt_demo.zzz_win_loss_records": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_win_loss_records", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_win_loss_records.sql", "original_file_path": "src/models/sql_model1/zzz_win_loss_records.sql", "unique_id": "model.dbt_demo.zzz_win_loss_records", "fqn": ["dbt_demo", "sql_model1", "zzz_win_loss_records"], "alias": "zzz_win_loss_records", "checksum": {"name": "sha256", "checksum": "2af4334f4afcfc71dbb39dd2bb88e64cf6e49fb20ec401ef0abf345aa827eb69"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {"team": {"name": "team", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "wins": {"name": "wins", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "losses": {"name": "losses", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733493328.3391528, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_win_loss_records`", "raw_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from {{ ref('zzz_game_details') }}\n)\ngroup by winner\norder by wins desc", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "compiled": true, "compiled_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from `dbt_demo`.`dbt_dev`.`zzz_game_details`\n)\ngroup by winner\norder by wins desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_win_loss_records_team"], "alias": "unique_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3774579, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "unique", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_team"], "alias": "not_null_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.378356, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_wins", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_wins.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_wins"], "alias": "not_null_zzz_win_loss_records_wins", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3790798, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "wins", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "wins", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_losses", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_losses.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_losses"], "alias": "not_null_zzz_win_loss_records_losses", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.379801, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "losses", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "losses", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}}, "sources": {"source.dbt_demo.raw.diamonds": {"database": "hive_metastore", "schema": "default", "name": "diamonds", "resource_type": "source", "package_name": "dbt_demo", "path": "src/models/sql_model1/sources.yml", "original_file_path": "src/models/sql_model1/sources.yml", "unique_id": "source.dbt_demo.raw.diamonds", "fqn": ["dbt_demo", "sql_model1", "raw", "diamonds"], "source_name": "raw", "source_description": "", "loader": "", "identifier": "diamonds", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "`hive_metastore`.`default`.`diamonds`", "created_at": 1733486794.541405}}, "macros": {"macro.dbt_demo.cents_to_dollars": {"name": "cents_to_dollars", "resource_type": "macro", "package_name": "dbt_demo", "path": "src/macros/centrs_to_dollars.sql", "original_file_path": "src/macros/centrs_to_dollars.sql", "unique_id": "macro.dbt_demo.cents_to_dollars", "macro_sql": "{% macro cents_to_dollars(column_name, scale=2) %}\n ({{ column_name }} / 100)::numeric(16, {{ scale }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.526016, "supported_languages": null}, "macro.dbt_databricks.materialization_materialized_view_databricks": {"name": "materialization_materialized_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialization_materialized_view_databricks", "macro_sql": "{% materialization materialized_view, adapter = 'databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.materialized_view_get_build_sql", "macro.dbt_databricks.materialized_view_execute_no_op", "macro.dbt_databricks.materialized_view_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5290601, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt_databricks.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.530187, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5303931, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.531114, "supported_languages": null}, "macro.dbt_databricks.materialization_view_databricks": {"name": "materialization_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.materialization_view_databricks", "macro_sql": "{% materialization view, adapter='databricks' -%}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and not old_relation.is_view -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.532544, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_table_databricks": {"name": "materialization_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_databricks.materialization_table_databricks", "macro_sql": "{% materialization table, adapter = 'databricks', supported_languages=['sql', 'python'] %}\n {{ log(\"MATERIALIZING TABLE\") }}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation and (not (old_relation.is_delta and config.get('file_format', default='delta') == 'delta')) or (old_relation.is_materialized_view or old_relation.is_streaming_table) -%}\n {{ adapter.drop_relation(old_relation) }}\n {%- endif %}\n\n -- build model\n\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% if language==\"python\" %}\n {% do apply_tblproperties(target_relation, tblproperties) %}\n {% endif %}\n {%- do apply_tags(target_relation, tags) -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.apply_tags", "macro.dbt.persist_docs", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5349078, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks_build_snapshot_staging_table": {"name": "databricks_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro_sql": "{% macro databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=target_relation.database,\n type='view') -%}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n create or replace view {{ tmp_relation }}\n as\n {{ select }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5406778, "supported_languages": null}, "macro.dbt_databricks.materialization_snapshot_databricks": {"name": "materialization_snapshot_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.materialization_snapshot_databricks", "macro_sql": "{% materialization snapshot, adapter='databricks' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'delta') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = databricks__get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table',\n needs_information=True) -%}\n\n {%- if file_format not in ['delta', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% if target_relation.database is none %}\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% else %}\n {% set staging_table = databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% endif %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% call statement_with_staging_table('main', staging_table) %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=True) %}\n\n {% endif %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.544019, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_streaming_table_databricks": {"name": "materialization_streaming_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.materialization_streaming_table_databricks", "macro_sql": "{% materialization streaming_table, adapter='databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.StreamingTable) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% set build_sql = streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ streaming_table_execute_no_op(target_relation) }}\n {% else %}\n {{ streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.streaming_table_get_build_sql", "macro.dbt_databricks.streaming_table_execute_no_op", "macro.dbt_databricks.streaming_table_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.546883, "supported_languages": ["sql"]}, "macro.dbt_databricks.streaming_table_get_build_sql": {"name": "streaming_table_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_get_build_sql", "macro_sql": "{% macro streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_streaming_table_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_streaming_table %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_streaming_table_configuration_changes(existing_relation, config) %}\n {% if configuration_changes is none %}\n {% set build_sql = refresh_streaming_table(target_relation, sql) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_streaming_table_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro.dbt_databricks.refresh_streaming_table", "macro.dbt_databricks.get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5479271, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_no_op": {"name": "streaming_table_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_no_op", "macro_sql": "{% macro streaming_table_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548121, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_build_sql": {"name": "streaming_table_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_build_sql", "macro_sql": "{% macro streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548807, "supported_languages": null}, "macro.dbt_databricks.databricks__can_clone_table": {"name": "databricks__can_clone_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__can_clone_table", "macro_sql": "{% macro databricks__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550354, "supported_languages": null}, "macro.dbt_databricks.databricks__create_or_replace_clone": {"name": "databricks__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__create_or_replace_clone", "macro_sql": "{% macro databricks__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace\n table {{ this_relation }}\n shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550466, "supported_languages": null}, "macro.dbt_databricks.materialization_clone_databricks": {"name": "materialization_clone_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.materialization_clone_databricks", "macro_sql": "\n\n{%- materialization clone, adapter='databricks' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.552143, "supported_languages": ["sql"]}, "macro.dbt_databricks.dbt_databricks_validate_get_file_format": {"name": "dbt_databricks_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro_sql": "{% macro dbt_databricks_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.553088, "supported_languages": null}, "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy": {"name": "dbt_databricks_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'merge', 'replace_where', 'append', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_delta_only_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via warehouse\n Use the 'merge' or 'replace_where' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite', 'replace_where'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy == 'replace_where' and file_format not in ['delta'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55381, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_default_sql": {"name": "databricks__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_default_sql", "macro_sql": "{% macro databricks__get_incremental_default_sql(arg_dict) %}\n {{ return(get_incremental_merge_sql(arg_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55716, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_append_sql": {"name": "databricks__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_append_sql", "macro_sql": "{% macro databricks__get_incremental_append_sql(arg_dict) %}\n {% do return(get_insert_into_sql(arg_dict[\"temp_relation\"], arg_dict[\"target_relation\"])) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5573308, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_replace_where_sql": {"name": "databricks__get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_replace_where_sql", "macro_sql": "{% macro databricks__get_incremental_replace_where_sql(arg_dict) %}\n {% do return(get_replace_where_sql(arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557461, "supported_languages": null}, "macro.dbt_databricks.get_incremental_replace_where_sql": {"name": "get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_incremental_replace_where_sql", "macro_sql": "{% macro get_incremental_replace_where_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_replace_where_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557615, "supported_languages": null}, "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql": {"name": "databricks__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql", "macro_sql": "{% macro databricks__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}\n {{ return(get_insert_overwrite_sql(source, target)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5577788, "supported_languages": null}, "macro.dbt_databricks.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558079, "supported_languages": null}, "macro.dbt_databricks.get_replace_where_sql": {"name": "get_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_replace_where_sql", "macro_sql": "{% macro get_replace_where_sql(args_dict) -%}\n {%- set predicates = args_dict['incremental_predicates'] -%}\n {%- set target_relation = args_dict['target_relation'] -%}\n {%- set temp_relation = args_dict['temp_relation'] -%}\n\n insert into {{ target_relation }}\n {% if predicates %}\n {% if predicates is sequence and predicates is not string %}\n replace where {{ predicates | join(' and ') }}\n {% else %}\n replace where {{ predicates }}\n {% endif %}\n {% endif %}\n table {{ temp_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5584981, "supported_languages": null}, "macro.dbt_databricks.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) | map(attribute=\"quoted\") | list -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) | map(attribute=\"quoted\") | list -%}\n {{ insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.insert_into_sql_impl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558819, "supported_languages": null}, "macro.dbt_databricks.insert_into_sql_impl": {"name": "insert_into_sql_impl", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.insert_into_sql_impl", "macro_sql": "{% macro insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) %}\n {%- set common_columns = [] -%}\n {%- for dest_col in dest_columns -%}\n {%- if dest_col in source_columns -%}\n {%- do common_columns.append(dest_col) -%}\n {%- else -%}\n {%- do common_columns.append('DEFAULT') -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set dest_cols_csv = dest_columns | join(', ') -%}\n {%- set source_cols_csv = common_columns | join(', ') -%}\ninsert into table {{ target_relation }} ({{ dest_cols_csv }})\nselect {{source_cols_csv}} from {{ source_relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5592902, "supported_languages": null}, "macro.dbt_databricks.databricks__get_merge_sql": {"name": "databricks__get_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_merge_sql", "macro_sql": "{% macro databricks__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set source_columns = (adapter.get_columns_in_relation(source) | map(attribute='quoted') | list)-%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} <=> DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} <=> DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n when matched then update set {{ get_merge_update_set(update_columns, on_schema_change, source_columns) }}\n when not matched then insert {{ get_merge_insert(on_schema_change, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.incremental_validate_on_schema_change", "macro.dbt.get_merge_update_columns", "macro.dbt_databricks.get_merge_update_set", "macro.dbt_databricks.get_merge_insert"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5605829, "supported_languages": null}, "macro.dbt_databricks.get_merge_update_set": {"name": "get_merge_update_set", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_update_set", "macro_sql": "{% macro get_merge_update_set(update_columns, on_schema_change, source_columns) %}\n {%- if update_columns -%}\n {%- for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- elif on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n {%- for column in source_columns -%}\n {{ column }} = DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.560978, "supported_languages": null}, "macro.dbt_databricks.get_merge_insert": {"name": "get_merge_insert", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_insert", "macro_sql": "{% macro get_merge_insert(on_schema_change, source_columns) %}\n {%- if on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n ({{ source_columns | join(\", \") }}) VALUES (\n {%- for column in source_columns -%}\n DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %})\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.561265, "supported_languages": null}, "macro.dbt_databricks.materialization_incremental_databricks": {"name": "materialization_incremental_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.materialization_incremental_databricks", "macro_sql": "{% materialization incremental, adapter='databricks', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='delta') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'merge' -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n {%- set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool -%}\n\n {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%}\n {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set incremental_predicates = config.get('predicates', default=none) or config.get('incremental_predicates', default=none) -%}\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%}\n {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%}\n {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%}\n {%- else -%}\n {%- set temp_relation_suffix = '__dbt_tmp' -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode to STATIC for initial replace --#}\n {%- if incremental_strategy == 'insert_overwrite' and should_full_refresh() -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = STATIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {% do apply_tags(target_relation, tags) %}\n {%- if language == 'python' -%}\n {%- do apply_tblproperties(target_relation, tblproperties) %}\n {%- endif -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- elif existing_relation.is_view or existing_relation.is_materialized_view or existing_relation.is_streaming_table or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% if not existing_relation.is_view %}\n {% do persist_constraints(target_relation, model) %}\n {% endif %}\n {% do apply_tags(target_relation, tags) %}\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- else -%}\n {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n {#-- Relation must be merged --#}\n {%- set _existing_config = adapter.get_relation_config(existing_relation) -%}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%}\n {%- set temp_relation = databricks__make_temp_relation(target_relation, suffix=temp_relation_suffix, as_table=language != 'sql') -%}\n {%- call statement('create_temp_relation', language=language) -%}\n {{ create_table_as(True, temp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, temp_relation, existing_relation) -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': temp_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {%- set build_sql = strategy_sql_macro_func(strategy_arg_dict) -%}\n {%- if language == 'sql' -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', temp_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why does not either drop_relation or adapter.drop_relation work here?!\n --#}\n {%- endif -%}\n {% do apply_liquid_clustered_cols(target_relation) %}\n {% if _configuration_changes is not none %}\n {% set tags = _configuration_changes.changes.get(\"tags\", None) %}\n {% set tblproperties = _configuration_changes.changes.get(\"tblproperties\", None) %}\n {% if tags is not none %}\n {% do apply_tags(target_relation, tags.set_tags, tags.unset_tags) %}\n {%- endif -%}\n {% if tblproperties is not none %}\n {% do apply_tblproperties(target_relation, tblproperties.tblproperties) %}\n {%- endif -%}\n {%- endif -%}\n {% do persist_docs(target_relation, model, for_relation=True) %}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt.persist_docs", "macro.dbt_databricks.databricks__make_temp_relation", "macro.dbt.process_schema_changes", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt_databricks.apply_liquid_clustered_cols", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.566802, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks__get_binding_char": {"name": "databricks__get_binding_char", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__get_binding_char", "macro_sql": "{% macro databricks__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5697222, "supported_languages": null}, "macro.dbt_databricks.databricks__load_csv_rows": {"name": "databricks__load_csv_rows", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__load_csv_rows", "macro_sql": "{% macro databricks__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n {% set must_cast = model['config'].get(\"file_format\", \"delta\") == \"parquet\" %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- if must_cast -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- else -%}\n {{ get_binding_char() }}\n {%- endif -%}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True, close_cursor=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.571152, "supported_languages": null}, "macro.dbt_databricks.databricks__reset_csv_table": {"name": "databricks__reset_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__reset_csv_table", "macro_sql": "{% macro databricks__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {% if old_relation.is_delta and config.get('file_format', default='delta') == 'delta' %}\n {% set sql = create_or_replace_csv_table(model, agate_table, True) %}\n {% else %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {% else %}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table", "macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57173, "supported_languages": null}, "macro.dbt_databricks.create_or_replace_csv_table": {"name": "create_or_replace_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.create_or_replace_csv_table", "macro_sql": "{% macro create_or_replace_csv_table(model, agate_table, replace=False) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {%- set column_comment = config.persist_column_docs() and model.columns %}\n {%- set identifier = model['alias'] -%}\n {%- set relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}\n {%- set replace_clause = \"\" -%}\n {%- if replace -%}\n {%- set replace_clause = \"or replace\" -%}\n {%- endif -%}\n\n {% set sql %}\n create {{replace_clause}} table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {%- set column_comment_clause = \"\" -%}\n {%- if column_comment and col_name in model.columns.keys() -%} \n {%- set comment = model.columns[col_name]['description'] | replace(\"'\", \"\\\\'\") -%}\n {%- if comment and comment != \"\" -%}\n {%- set column_comment_clause = \"comment '\" ~ comment ~ \"'\" -%}\n {%- endif -%}\n {%- endif -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {{ column_comment_clause }}{%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57331, "supported_languages": null}, "macro.dbt_databricks.databricks__create_csv_table": {"name": "databricks__create_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__create_csv_table", "macro_sql": "{% macro databricks__create_csv_table(model, agate_table) %}\n {{ return(create_or_replace_csv_table(model, agate_table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.573471, "supported_languages": null}, "macro.dbt_databricks.materialization_seed_databricks": {"name": "materialization_seed_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.materialization_seed_databricks", "macro_sql": "{% materialization seed, adapter='databricks' %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and (old_relation.is_view or old_relation.is_materialized_view)) -%}\n {%- set exists_as_streaming_table = (old_relation is not none and old_relation.is_streaming_table) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view or a materialized view\".format(old_relation)) }}\n {% elif exists_as_streaming_table %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a streaming table\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n -- No need to persist docs, already handled in seed create\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576501, "supported_languages": ["sql"]}, "macro.dbt_databricks.databricks__generate_database_name": {"name": "databricks__generate_database_name", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt_databricks.databricks__generate_database_name", "macro_sql": "{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n {{ return(default_database) }}\n {%- else -%}\n {{ return(custom_database_name) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576849, "supported_languages": null}, "macro.dbt_databricks.databricks__get_drop_sql": {"name": "databricks__get_drop_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__get_drop_sql", "macro_sql": "{% macro databricks__get_drop_sql(relation) -%}\n {%- if relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n {%- elif relation.is_streaming_table-%}\n {{ drop_streaming_table(relation) }}\n {%- elif relation.is_view -%}\n {{ drop_view(relation) }}\n {%- else -%}\n {{ drop_table(relation) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_materialized_view", "macro.dbt_databricks.drop_streaming_table", "macro.dbt.drop_view", "macro.dbt.drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577309, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_relation": {"name": "databricks__drop_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_relation", "macro_sql": "{% macro databricks__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577478, "supported_languages": null}, "macro.dbt_databricks.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {% do return(adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57809, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_sql": {"name": "databricks__get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_sql", "macro_sql": "{% macro databricks__get_replace_sql(existing_relation, target_relation, sql) %}\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_create_backup_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_drop_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n ]) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ return([\n get_create_backup_sql(existing_relation),\n get_create_sql(target_relation, sql),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ return([\n get_drop_sql(existing_relation),\n get_create_sql(target_relation, sql)\n ]) }}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.579058, "supported_languages": null}, "macro.dbt_databricks.fetch_tags": {"name": "fetch_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags", "macro_sql": "{% macro fetch_tags(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {% call statement('list_tags', fetch_result=True) -%}\n {{ fetch_tags_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_tags').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_tags_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5802681, "supported_languages": null}, "macro.dbt_databricks.fetch_tags_sql": {"name": "fetch_tags_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags_sql", "macro_sql": "{% macro fetch_tags_sql(relation) -%}\n SELECT tag_name, tag_value\n FROM `system`.`information_schema`.`table_tags`\n WHERE catalog_name = '{{ relation.database|lower }}' \n AND schema_name = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5804422, "supported_languages": null}, "macro.dbt_databricks.apply_tags": {"name": "apply_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.apply_tags", "macro_sql": "{% macro apply_tags(relation, set_tags, unset_tags=[]) -%}\n {%- if (set_tags or unset_tags) and relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif -%}\n {%- if set_tags %}\n {%- call statement('set_tags') -%}\n {{ alter_set_tags(relation, set_tags) }}\n {%- endcall -%}\n {%- endif %}\n {%- if unset_tags %}\n {%- call statement('unset_tags') -%}\n {{ alter_unset_tags(relation, unset_tags) }}\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_set_tags", "macro.dbt_databricks.alter_unset_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.580887, "supported_languages": null}, "macro.dbt_databricks.alter_set_tags": {"name": "alter_set_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_set_tags", "macro_sql": "{% macro alter_set_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} SET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' = '{{ tags[tag] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.581147, "supported_languages": null}, "macro.dbt_databricks.alter_unset_tags": {"name": "alter_unset_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_unset_tags", "macro_sql": "{% macro alter_unset_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} UNSET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n )\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58137, "supported_languages": null}, "macro.dbt_databricks.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/location.sql", "original_file_path": "macros/relations/location.sql", "unique_id": "macro.dbt_databricks.location_clause", "macro_sql": "{% macro location_clause(relation) %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- elif (not relation.is_hive_metastore()) and file_format != 'delta' -%}\n {{ exceptions.raise_compiler_error(\n 'Incompatible configuration: `location_root` must be set when using a non-delta file format with Unity Catalog'\n ) }}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5819929, "supported_languages": null}, "macro.dbt_databricks.optimize": {"name": "optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.optimize", "macro_sql": "{% macro optimize(relation) %}\n {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.582578, "supported_languages": null}, "macro.dbt_databricks.databricks__optimize": {"name": "databricks__optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.databricks__optimize", "macro_sql": "\n\n{%- macro databricks__optimize(relation) -%}\n {%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and\n var('databricks_skip_optimize', 'false')|lower != 'true' and\n config.get('file_format', 'delta') == 'delta' -%}\n {%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) -%}\n {%- call statement('run_optimize_stmt') -%}\n {{ get_optimize_sql(relation) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_optimize_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583037, "supported_languages": null}, "macro.dbt_databricks.get_optimize_sql": {"name": "get_optimize_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.get_optimize_sql", "macro_sql": "{%- macro get_optimize_sql(relation) %}\n optimize {{ relation }}\n {%- if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}\n {%- if config.get('liquid_clustered_by', False) %}\n {{ exceptions.warn(\"Both zorder and liquid_clustered_by are set but they are incompatible. zorder will be ignored.\") }}\n {%- else %}\n {%- set zorder = config.get('zorder', none) %}\n {# TODO: predicates here? WHERE ... #}\n {%- if zorder is sequence and zorder is not string %}\n zorder by (\n {%- for col in zorder %}\n {{ col }}{% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- else %}\n zorder by ({{zorder}})\n {%- endif %}\n {%- endif %}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583699, "supported_languages": null}, "macro.dbt_databricks.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/file_format.sql", "original_file_path": "macros/relations/file_format.sql", "unique_id": "macro.dbt_databricks.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {%- set file_format = config.get('file_format', default='delta') -%}\n using {{ file_format }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583901, "supported_languages": null}, "macro.dbt_databricks.liquid_clustered_cols": {"name": "liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.liquid_clustered_cols", "macro_sql": "{% macro liquid_clustered_cols() -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n CLUSTER BY ({{ cols | join(', ') }})\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5845, "supported_languages": null}, "macro.dbt_databricks.apply_liquid_clustered_cols": {"name": "apply_liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.apply_liquid_clustered_cols", "macro_sql": "{% macro apply_liquid_clustered_cols(target_relation) -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {%- call statement('set_cluster_by_columns') -%}\n ALTER {{ target_relation.type }} {{ target_relation }} CLUSTER BY ({{ cols | join(', ') }})\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.584919, "supported_languages": null}, "macro.dbt_databricks.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() -%}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58541, "supported_languages": null}, "macro.dbt_databricks.databricks__tblproperties_clause": {"name": "databricks__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.databricks__tblproperties_clause", "macro_sql": "{% macro databricks__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5857081, "supported_languages": null}, "macro.dbt_databricks.apply_tblproperties": {"name": "apply_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.apply_tblproperties", "macro_sql": "{% macro apply_tblproperties(relation, tblproperties) -%}\n {% if tblproperties %}\n {%- call statement('apply_tblproperties') -%}\n ALTER {{ relation.type }} {{ relation }} SET TBLPROPERTIES (\n {% for tblproperty in tblproperties -%}\n '{{ tblproperty }}' = '{{ tblproperties[tblproperty] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n {%- endcall -%}\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58604, "supported_languages": null}, "macro.dbt_databricks.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.599794, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_constraints": {"name": "databricks__persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__persist_constraints", "macro_sql": "{% macro databricks__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% set has_model_contract = contract_config and contract_config.enforced %}\n {% set has_databricks_constraints = config.get('persist_constraints', False) %}\n\n {% if (has_model_contract or has_databricks_constraints) %}\n {% if config.get('file_format', 'delta') != 'delta' %}\n {# Constraints are only supported for delta tables #}\n {{ exceptions.warn(\"Constraints not supported for file format: \" ~ config.get('file_format')) }}\n {% elif relation.is_view %}\n {# Constraints are not supported for views. This point in the code should not have been reached. #}\n {{ exceptions.raise_compiler_error(\"Constraints not supported for views.\") }}\n {% elif is_incremental() %}\n {# Constraints are not applied for incremental updates. This point in the code should not have been reached #}\n {{ exceptions.raise_compiler_error(\"Constraints are not applied for incremental updates. Full refresh is required to update constraints.\") }}\n {% else %}\n {% do alter_column_set_constraints(relation, model) %}\n {% do alter_table_add_constraints(relation, model) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt_databricks.alter_column_set_constraints", "macro.dbt_databricks.alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600513, "supported_languages": null}, "macro.dbt_databricks.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600693, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_table_add_constraints": {"name": "databricks__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_table_add_constraints", "macro_sql": "{% macro databricks__alter_table_add_constraints(relation, model) %}\n {% set constraints = get_model_constraints(model) %}\n {% set statements = get_constraints_sql(relation, constraints, model) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_model_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6010072, "supported_languages": null}, "macro.dbt_databricks.get_model_constraints": {"name": "get_model_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_model_constraints", "macro_sql": "{% macro get_model_constraints(model) %}\n {% set constraints = model.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and model.get('meta', {}).get('constraints') is sequence %}\n {# Databricks constraints implementation. Constraints are in the meta property. #}\n {% set db_constraints = model.get('meta', {}).get('constraints', []) %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.601491, "supported_languages": null}, "macro.dbt_databricks.get_column_constraints": {"name": "get_column_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_column_constraints", "macro_sql": "{% macro get_column_constraints(column) %}\n {% set constraints = column.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and column.get('meta', {}).get('constraint') %}\n {# Databricks constraints implementation. Constraint is in the meta property. #}\n {% set db_constraints = [column.get('meta', {}).get('constraint')] %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints, column) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6019762, "supported_languages": null}, "macro.dbt_databricks.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602155, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_set_constraints": {"name": "databricks__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_set_constraints", "macro_sql": "{% macro databricks__alter_column_set_constraints(relation, model) %}\n {% set column_dict = model.columns %}\n {% for column_name in column_dict %}\n {% set column = column_dict[column_name] %}\n {% set constraints = get_column_constraints(column) %}\n {% set statements = get_constraints_sql(relation, constraints, model, column) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602596, "supported_languages": null}, "macro.dbt_databricks.get_constraints_sql": {"name": "get_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraints_sql", "macro_sql": "{% macro get_constraints_sql(relation, constraints, model, column={}) %}\n {% set statements = [] %}\n -- Hack so that not null constraints will be applied before primary key constraints\n {% for constraint in constraints|sort(attribute='type') %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endfor %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_constraint_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.603168, "supported_languages": null}, "macro.dbt_databricks.get_constraint_sql": {"name": "get_constraint_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraint_sql", "macro_sql": "{% macro get_constraint_sql(relation, constraint, model, column={}) %}\n {% set statements = [] %}\n {% set type = constraint.get(\"type\", \"\") %}\n\n {% if type == 'check' %}\n {% set expression = constraint.get(\"expression\", \"\") %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Invalid check constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5 (relation.identifier ~ \";\" ~ column.get(\"name\", \"\") ~ \";\" ~ expression ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" check (\" ~ expression ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'not_null' %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if column %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% set stmt = \"alter table \" ~ relation ~ \" change column \" ~ quoted_name ~ \" set not null \" ~ (constraint.expression or \"\") ~ \";\" %}\n {% do statements.append(stmt) %}\n {% else %}\n {{ exceptions.warn('not_null constraint on invalid column: ' ~ column_name) }}\n {% endif %}\n {% endfor %}\n {% elif type == 'primary_key' %}\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ type)}}\n {% endif %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid primary key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"primary_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" primary key(\" ~ joined_names ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'foreign_key' %}\n\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n \n {% if constraint.get('expression') %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ constraint.get('expression') ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key\" ~ constraint.get('expression') %}\n {% else %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid foreign key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set parent = constraint.get(\"to\") %}\n {% if not parent %}\n {{ exceptions.raise_compiler_error('No parent table defined for foreign key: ' ~ expression) }}\n {% endif %}\n {% if not \".\" in parent %}\n {% set parent = relation.schema ~ \".\" ~ parent%}\n {% endif %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\" ~ parent ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key(\" ~ joined_names ~ \") references \" ~ parent %}\n {% set parent_columns = constraint.get(\"to_columns\") %}\n {% if parent_columns %}\n {% set stmt = stmt ~ \"(\" ~ parent_columns|join(\", \") ~ \")\"%}\n {% endif %}\n {% endif %}\n {% set stmt = stmt ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif constraint.get('warn_unsupported') %}\n {{ exceptions.warn(\"unsupported constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6086242, "supported_languages": null}, "macro.dbt_databricks.databricks_constraints_to_dbt": {"name": "databricks_constraints_to_dbt", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks_constraints_to_dbt", "macro_sql": "{% macro databricks_constraints_to_dbt(constraints, column) %}\n {# convert constraints defined using the original databricks format #}\n {% set dbt_constraints = [] %}\n {% for constraint in constraints %}\n {% if constraint.get and constraint.get(\"type\") %}\n {# already in model contract format #}\n {% do dbt_constraints.append(constraint) %}\n {% else %}\n {% if column %}\n {% if constraint == \"not_null\" %}\n {% do dbt_constraints.append({\"type\": \"not_null\", \"columns\": [column.get(\"name\")]}) %}\n {% else %}\n {{ exceptions.raise_compiler_error('Invalid constraint for column ' ~ column.get(\"name\", \"\") ~ '. Only `not_null` is supported.') }}\n {% endif %}\n {% else %}\n {% set name = constraint['name'] %}\n {% if not name %}\n {{ exceptions.raise_compiler_error('Invalid check constraint name') }}\n {% endif %}\n {% set condition = constraint['condition'] %}\n {% if not condition %}\n {{ exceptions.raise_compiler_error('Invalid check constraint condition') }}\n {% endif %}\n {% do dbt_constraints.append({\"name\": name, \"type\": \"check\", \"expression\": condition}) %}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(dbt_constraints) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.609767, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_sql": {"name": "databricks__get_create_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_sql", "macro_sql": "{%- macro databricks__get_create_sql(relation, sql) -%}\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_streaming_table -%}\n {{ get_create_streaming_table_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610386, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_materialized_view": {"name": "databricks__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_materialized_view", "macro_sql": "{% macro databricks__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610508, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_materialized_view": {"name": "databricks__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_materialized_view", "macro_sql": "{% macro databricks__refresh_materialized_view(relation) -%}\n refresh materialized view {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61062, "supported_languages": null}, "macro.dbt_databricks.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611409, "supported_languages": null}, "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes": {"name": "databricks__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes", "macro_sql": "\n\n\n{%- macro databricks__get_materialized_view_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_materialized_view = adapter.get_relation_config(existing_relation) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = materialized_view.get_changeset(_existing_materialized_view) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6116831, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql": {"name": "databricks__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {% do return(get_alter_mv_internal(relation, configuration_changes)) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_mv_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611993, "supported_languages": null}, "macro.dbt_databricks.get_alter_mv_internal": {"name": "get_alter_mv_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_mv_internal", "macro_sql": "{% macro get_alter_mv_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n -- Currently only schedule can be altered\n ALTER MATERIALIZED VIEW {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, refresh.is_altered) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6122189, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql": {"name": "databricks__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_create_materialized_view_as_sql(relation, sql) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = materialized_view.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = materialized_view.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = materialized_view.config[\"comment\"].comment -%}\n {%- set refresh = materialized_view.config[\"refresh\"] -%}\n create materialized view {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.612938, "supported_languages": null}, "macro.dbt_databricks.drop_streaming_table": {"name": "drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.drop_streaming_table", "macro_sql": "{% macro drop_streaming_table(relation) -%}\n {{ return(adapter.dispatch('drop_streaming_table', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.default__drop_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.613157, "supported_languages": null}, "macro.dbt_databricks.default__drop_streaming_table": {"name": "default__drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.default__drop_streaming_table", "macro_sql": "{% macro default__drop_streaming_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6132412, "supported_languages": null}, "macro.dbt_databricks.refresh_streaming_table": {"name": "refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.refresh_streaming_table", "macro_sql": "{% macro refresh_streaming_table(relation, sql) -%}\n {{ adapter.dispatch('refresh_streaming_table', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61346, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_streaming_table": {"name": "databricks__refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_streaming_table", "macro_sql": "{% macro databricks__refresh_streaming_table(relation, sql) -%}\n create or refresh streaming table {{ relation }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6135712, "supported_languages": null}, "macro.dbt_databricks.get_alter_streaming_table_as_sql": {"name": "get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_streaming_table_as_sql", "macro_sql": "{% macro get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_streaming_table_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.615469, "supported_languages": null}, "macro.dbt_databricks.get_streaming_table_configuration_changes": {"name": "get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro_sql": "{% macro get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_streaming_table_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_streaming_table_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6158211, "supported_languages": null}, "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes": {"name": "databricks__get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes", "macro_sql": "\n\n{%- macro databricks__get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_streaming_table = adapter.get_relation_config(existing_relation) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = streaming_table.get_changeset(_existing_streaming_table) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6161752, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql": {"name": "databricks__get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {%- set alter_statement = get_alter_st_internal(relation, configuration_changes) -%}\n {%- set create_statement = get_create_st_internal(relation, configuration_changes, sql) -%}\n {%- set return_statements = [] -%}\n {%- if create_statement -%}\n {{ return_statements.append(create_statement) }}\n {%- endif -%}\n {%- if alter_statement -%}\n {{ return_statements.append(alter_statement) }}\n {%- endif -%}\n {% do return(return_statements) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_st_internal", "macro.dbt_databricks.get_create_st_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.616995, "supported_languages": null}, "macro.dbt_databricks.get_create_st_internal": {"name": "get_create_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_create_st_internal", "macro_sql": "{% macro get_create_st_internal(relation, configuration_changes, sql) %}\n {%- set partition_by = configuration_changes.changes[\"partition_by\"].partition_by -%}\n {%- set tblproperties = configuration_changes.changes[\"tblproperties\"].tblproperties -%}\n {%- set comment = configuration_changes.changes[\"comment\"].comment -%}\n CREATE OR REFRESH STREAMING TABLE {{ relation }}\n {% if partition_by -%}\n {{ get_create_sql_partition_by(partition_by) }}\n {%- endif %}\n {% if comment -%}\n {{ get_create_sql_comment(comment) }}\n {%- endif %}\n {% if tblproperties -%}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {%- endif %}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6175182, "supported_languages": null}, "macro.dbt_databricks.get_alter_st_internal": {"name": "get_alter_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_st_internal", "macro_sql": "{% macro get_alter_st_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n {%- if refresh and refresh.cron -%}\n ALTER STREAMING TABLE {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, False) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6177878, "supported_languages": null}, "macro.dbt_databricks.get_create_streaming_table_as_sql": {"name": "get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro_sql": "{% macro get_create_streaming_table_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_streaming_table_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618218, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql": {"name": "databricks__get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_create_streaming_table_as_sql(relation, sql) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = streaming_table.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = streaming_table.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = streaming_table.config[\"comment\"].comment -%}\n {%- set refresh = streaming_table.config[\"refresh\"] -%}\n\n CREATE STREAMING TABLE {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618762, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_comment": {"name": "get_create_sql_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/comment.sql", "original_file_path": "macros/relations/components/comment.sql", "unique_id": "macro.dbt_databricks.get_create_sql_comment", "macro_sql": "{%- macro get_create_sql_comment(comment) -%}\n{% if comment is string -%}\n COMMENT '{{ comment }}'\n{%- endif -%}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6189408, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_refresh_schedule": {"name": "get_create_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_create_sql_refresh_schedule", "macro_sql": "{% macro get_create_sql_refresh_schedule(cron, time_zone_value) %}\n {%- if cron -%}\n SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619374, "supported_languages": null}, "macro.dbt_databricks.get_alter_sql_refresh_schedule": {"name": "get_alter_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_alter_sql_refresh_schedule", "macro_sql": "{% macro get_alter_sql_refresh_schedule(cron, time_zone_value, is_altered) %}\n {%- if cron -%}\n {%- if is_altered -%}\n ALTER SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- else -%}\n ADD SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n {%- else -%}\n DROP SCHEDULE\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619678, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_partition_by": {"name": "get_create_sql_partition_by", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/partitioning.sql", "original_file_path": "macros/relations/components/partitioning.sql", "unique_id": "macro.dbt_databricks.get_create_sql_partition_by", "macro_sql": "{% macro get_create_sql_partition_by(partition_by) -%}\n{%- if partition_by -%}\n PARTITIONED BY ({%- for col in partition_by -%}{{ col }}{% if not loop.last %}, {% endif %}{%- endfor %})\n{%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619945, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_tblproperties": {"name": "get_create_sql_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/tblproperties.sql", "original_file_path": "macros/relations/components/tblproperties.sql", "unique_id": "macro.dbt_databricks.get_create_sql_tblproperties", "macro_sql": "{% macro get_create_sql_tblproperties(tblproperties) %}\n {%- if tblproperties and tblproperties|length>0 -%}\n TBLPROPERTIES (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}'{%- if not loop.last -%}, {% endif -%}\n {% endfor -%}\n )\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62031, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_table": {"name": "databricks__drop_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_table", "macro_sql": "{% macro databricks__drop_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.620424, "supported_languages": null}, "macro.dbt_databricks.databricks__create_table_as": {"name": "databricks__create_table_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_table_as", "macro_sql": "{% macro databricks__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {% if file_format == 'delta' %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ liquid_clustered_cols() }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ databricks__py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_databricks.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.partition_cols", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt_databricks.databricks__py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6221719, "supported_languages": null}, "macro.dbt_databricks.databricks__options_clause": {"name": "databricks__options_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__options_clause", "macro_sql": "{% macro databricks__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format', default='delta') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.623204, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_view": {"name": "databricks__drop_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_view", "macro_sql": "{% macro databricks__drop_view(relation) -%}\n drop view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6233542, "supported_languages": null}, "macro.dbt_databricks.databricks__create_view_as": {"name": "databricks__create_view_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_view_as", "macro_sql": "{% macro databricks__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n {% if query_columns %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n {{ tblproperties_clause() }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_databricks.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent", "macro.dbt_databricks.tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6240902, "supported_languages": null}, "macro.dbt_databricks.statement_with_staging_table": {"name": "statement_with_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.statement_with_staging_table", "macro_sql": "{% macro statement_with_staging_table(name=None, staging_table=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result, staging_table=staging_table) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6248162, "supported_languages": null}, "macro.dbt_databricks.databricks__dateadd": {"name": "databricks__dateadd", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_databricks.databricks__dateadd", "macro_sql": "{% macro databricks__dateadd(datepart, interval, from_date_or_timestamp) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampadd({{datepart}}, {{interval}}, {{from_date_or_timestamp}})\n {%- else -%}\n {{ spark__dateadd(datepart, interval, from_date_or_timestamp) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6251829, "supported_languages": null}, "macro.dbt_databricks.databricks__datediff": {"name": "databricks__datediff", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_databricks.databricks__datediff", "macro_sql": "{% macro databricks__datediff(first_date, second_date, datepart) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampdiff({{datepart}}, {{date_trunc(datepart, first_date)}}, {{date_trunc(datepart, second_date)}})\n {%- else -%}\n {{ spark__datediff(first_date, second_date, datepart) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.date_trunc", "macro.dbt_spark.spark__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6255841, "supported_languages": null}, "macro.dbt_databricks.databricks_copy_into": {"name": "databricks_copy_into", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/copy_into.sql", "original_file_path": "macros/adapters/copy_into.sql", "unique_id": "macro.dbt_databricks.databricks_copy_into", "macro_sql": "{% macro databricks_copy_into(\n target_table,\n source,\n file_format,\n expression_list=none,\n source_credential=none,\n source_encryption=none,\n validate=none,\n files=none,\n pattern=none,\n format_options=none,\n copy_options=none) -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=target.database,\n schema=target.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- set source_clause -%}\n {%- if expression_list -%}\n ( select {{ expression_list }} from '{{ source }}' )\n {%- else -%}\n '{{ source }}'\n {%- endif -%}\n {%- if source_credential or source_encryption %}\n WITH (\n {%- if source_credential %}\n credential (\n {%- for name in source_credential -%}\n '{{ name }}' = '{{ source_credential[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {%- if source_encryption %}\n encryption (\n {%- for name in source_encryption -%}\n '{{ name }}' = '{{ source_encryption[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n )\n {%- endif -%}\n {%- endset -%}\n\n {% set query %}\n copy into {{ target_relation }}\n from {{ source_clause }}\n fileformat = {{ file_format }}\n {% if validate -%} validate {{ validate }} {%- endif %}\n {% if files and pattern %}\n {{ exceptions.raise_compiler_error(\"You can only specify one of 'files' or 'pattern'\") }}\n {% endif %}\n {% if files -%}\n files = (\n {%- for file in files -%}\n '{{ file }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if pattern -%}\n pattern = '{{ pattern }}'\n {%- endif %}\n {% if format_options -%}\n format_options (\n {%- for key in format_options -%}\n '{{ key }}' = '{{ format_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if copy_options -%}\n copy_options (\n {%- for key in copy_options -%}\n '{{ key }}' = '{{ copy_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% endset %}\n\n {% do log(\"Running COPY INTO\" ~ adapter.redact_credentials(query), info=True) %}\n {% do run_query(query) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62939, "supported_languages": null}, "macro.dbt_databricks.databricks__make_temp_relation": {"name": "databricks__make_temp_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__make_temp_relation", "macro_sql": "{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp', as_table=False) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {%- if as_table -%}\n {% set tmp_relation = api.Relation.create(\n identifier=tmp_identifier,\n schema=base_relation.schema,\n database=base_relation.database,\n type='table') %}\n {%- else -%}\n {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}\n {%- endif -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.630194, "supported_languages": null}, "macro.dbt_databricks.databricks__get_or_create_relation": {"name": "databricks__get_or_create_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__get_or_create_relation", "macro_sql": "{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}\n {%- set target_relation = adapter.get_relation(\n database=database,\n schema=schema,\n identifier=identifier,\n needs_information=needs_information) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6306949, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog": {"name": "databricks__get_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog", "macro_sql": "{% macro databricks__get_catalog(information_schema, schemas) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633368, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations": {"name": "databricks__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations", "macro_sql": "{% macro databricks__get_catalog_relations(information_schema, relations) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63369, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_tables_sql": {"name": "databricks__get_catalog_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro_sql": "{% macro databricks__get_catalog_tables_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n lower(table_type) as table_type,\n comment as table_comment,\n table_owner,\n 'Last Modified' as `stats:last_modified:label`,\n last_altered as `stats:last_modified:value`,\n 'The timestamp for last update/change' as `stats:last_modified:description`,\n (last_altered is not null and table_type not ilike '%VIEW%') as `stats:last_modified:include`\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6338282, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_columns_sql": {"name": "databricks__get_catalog_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro_sql": "{% macro databricks__get_catalog_columns_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n column_name,\n ordinal_position as column_index,\n lower(data_type) as column_type,\n comment as column_comment\n from `system`.`information_schema`.`columns`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633953, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_results_sql": {"name": "databricks__get_catalog_results_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro_sql": "{% macro databricks__get_catalog_results_sql() -%}\n select *\n from tables\n join columns using (table_database, table_schema, table_name)\n order by column_index\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.634028, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql": {"name": "databricks__get_catalog_schemas_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_schemas_where_clause_sql(catalog, schemas) -%}\n where table_catalog = '{{ catalog|lower }}' and ({%- for relation in schemas -%}\n table_schema = '{{ relation[1]|lower }}'{%- if not loop.last %} or {% endif -%}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63427, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql": {"name": "databricks__get_catalog_relations_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_relations_where_clause_sql(catalog, relations) -%}\n where table_catalog = '{{ catalog|lower }}' and (\n {%- for relation in relations -%}\n {% if relation.schema and relation.identifier %}\n (\n table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n )\n {% elif relation.schema %}\n (\n table_schema = '{{ relation.schema|lower }}'\n )\n {% else %}\n {% do exceptions.raise_compiler_error(\n '`get_catalog_relations` requires a list of relations, each with a schema'\n ) %}\n {% endif %}\n\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6348171, "supported_languages": null}, "macro.dbt_databricks.current_catalog": {"name": "current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.current_catalog", "macro_sql": "{% macro current_catalog() -%}\n {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__current_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6351402, "supported_languages": null}, "macro.dbt_databricks.databricks__current_catalog": {"name": "databricks__current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__current_catalog", "macro_sql": "{% macro databricks__current_catalog() -%}\n {% call statement('current_catalog', fetch_result=True) %}\n select current_catalog()\n {% endcall %}\n {% do return(load_result('current_catalog').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635343, "supported_languages": null}, "macro.dbt_databricks.use_catalog": {"name": "use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.use_catalog", "macro_sql": "{% macro use_catalog(catalog) -%}\n {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__use_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6355011, "supported_languages": null}, "macro.dbt_databricks.databricks__use_catalog": {"name": "databricks__use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__use_catalog", "macro_sql": "{% macro databricks__use_catalog(catalog) -%}\n {% call statement() %}\n use catalog {{ adapter.quote(catalog)|lower }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635657, "supported_languages": null}, "macro.dbt_databricks.databricks__py_write_table": {"name": "databricks__py_write_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.databricks__py_write_table", "macro_sql": "{% macro databricks__py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\nimport pyspark\n\n{{ py_try_import('pyspark.sql.connect.dataframe', 'newer_pyspark_available') }}\n{{ py_try_import('pandas', 'pandas_available') }}\n{{ py_try_import('pyspark.pandas', 'pyspark_pandas_api_available') }}\n{{ py_try_import('databricks.koalas', 'koalas_available') }}\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\n\nif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n if pyspark_pandas_api_available:\n df = pyspark.pandas.frame.DataFrame(df)\n elif koalas_available:\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif newer_pyspark_available and isinstance(df, pyspark.sql.connect.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\nwriter = (\n df.write\n .mode(\"overwrite\")\n .option(\"overwriteSchema\", \"true\")\n{{ py_get_writer_options()|indent(8, True) }}\n)\n\nwriter.saveAsTable(\"{{ target_relation }}\")\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.py_try_import", "macro.dbt_databricks.py_get_writer_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6376588, "supported_languages": null}, "macro.dbt_databricks.py_get_writer_options": {"name": "py_get_writer_options", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_get_writer_options", "macro_sql": "\n\n{%- macro py_get_writer_options() -%}\n{%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n{%- set file_format = config.get('file_format', validator=validation.any[basestring])|default('delta', true) -%}\n{%- set partition_by = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n{%- set liquid_clustered_by = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set clustered_by = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n.format(\"{{ file_format }}\")\n{%- if location_root is not none %}\n{%- set identifier = model['alias'] %}\n{%- if is_incremental() %}\n{%- set identifier = identifier + '__dbt_tmp' %}\n{%- endif %}\n.option(\"path\", \"{{ location_root }}/{{ identifier }}\")\n{%- endif -%}\n{%- if partition_by is not none -%}\n {%- if partition_by is string -%}\n {%- set partition_by = [partition_by] -%}\n {%- endif %}\n.partitionBy({{ partition_by }})\n{%- endif -%}\n{%- if liquid_clustered_by and not is_incremental() -%}\n {%- if liquid_clustered_by is string -%}\n {%- set liquid_clustered_by = [liquid_clustered_by] -%}\n {%- endif %}\n.clusterBy({{ liquid_clustered_by }})\n{%- endif -%}\n{%- if (clustered_by is not none) and (buckets is not none) -%}\n {%- if clustered_by is string -%}\n {%- set clustered_by = [clustered_by] -%}\n {%- endif %}\n.bucketBy({{ buckets }}, {{ clustered_by }})\n{%- endif -%}\n{% endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.is_incremental"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6390772, "supported_languages": null}, "macro.dbt_databricks.py_try_import": {"name": "py_try_import", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_try_import", "macro_sql": "{% macro py_try_import(library, var_name) -%}\n# make sure {{ library }} exists before using it\ntry:\n import {{ library }}\n {{ var_name }} = True\nexcept ImportError:\n {{ var_name }} = False\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.639234, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_comment": {"name": "databricks__alter_column_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_comment", "macro_sql": "{% macro databricks__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', default='delta') in ['delta', 'hudi'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641151, "supported_languages": null}, "macro.dbt_databricks.alter_table_comment": {"name": "alter_table_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_table_comment", "macro_sql": "{% macro alter_table_comment(relation, model) %}\n {% set comment_query %}\n comment on table {{ relation|lower }} is '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endset %}\n {% do run_query(comment_query) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641404, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments": {"name": "get_columns_comments", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_columns_comments", "macro_sql": "{% macro get_columns_comments(relation) -%}\n {% call statement('get_columns_comments', fetch_result=True) -%}\n describe table {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('get_columns_comments').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6416378, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_docs": {"name": "databricks__persist_docs", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__persist_docs", "macro_sql": "{% macro databricks__persist_docs(relation, model, for_relation, for_columns) -%}\n {%- if for_relation and config.persist_relation_docs() and model.description %}\n {% do alter_table_comment(relation, model) %}\n {% endif %}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {%- set existing_columns = adapter.get_columns_in_relation(relation) -%}\n {%- set columns_to_persist_docs = adapter.get_persist_doc_columns(existing_columns, model.columns) -%}\n {% do alter_column_comment(relation, columns_to_persist_docs) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.alter_table_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642109, "supported_languages": null}, "macro.dbt_databricks.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642476, "supported_languages": null}, "macro.dbt_databricks.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642708, "supported_languages": null}, "macro.dbt_databricks.databricks__list_relations_without_caching": {"name": "databricks__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__list_relations_without_caching", "macro_sql": "{% macro databricks__list_relations_without_caching(schema_relation) %}\n {{ return(adapter.get_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6461172, "supported_languages": null}, "macro.dbt_databricks.show_table_extended": {"name": "show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_table_extended", "macro_sql": "{% macro show_table_extended(schema_relation) %}\n {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_table_extended"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646267, "supported_languages": null}, "macro.dbt_databricks.databricks__show_table_extended": {"name": "databricks__show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_table_extended", "macro_sql": "{% macro databricks__show_table_extended(schema_relation) %}\n {% call statement('show_table_extended', fetch_result=True) -%}\n show table extended in {{ schema_relation.without_identifier()|lower }} like '{{ schema_relation.identifier|lower }}'\n {% endcall %}\n\n {% do return(load_result('show_table_extended').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646537, "supported_languages": null}, "macro.dbt_databricks.show_tables": {"name": "show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_tables", "macro_sql": "{% macro show_tables(relation) %}\n {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_tables"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646763, "supported_languages": null}, "macro.dbt_databricks.databricks__show_tables": {"name": "databricks__show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_tables", "macro_sql": "{% macro databricks__show_tables(relation) %}\n {% call statement('show_tables', fetch_result=True) -%}\n show tables in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646989, "supported_languages": null}, "macro.dbt_databricks.show_views": {"name": "show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_views", "macro_sql": "{% macro show_views(relation) %}\n {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_views"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647151, "supported_languages": null}, "macro.dbt_databricks.databricks__show_views": {"name": "databricks__show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_views", "macro_sql": "{% macro databricks__show_views(relation) %}\n {% call statement('show_views', fetch_result=True) -%}\n show views in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_views').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647396, "supported_languages": null}, "macro.dbt_databricks.databricks__get_relation_last_modified": {"name": "databricks__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__get_relation_last_modified", "macro_sql": "{% macro databricks__get_relation_last_modified(information_schema, relations) -%}\n\n {%- call statement('last_modified', fetch_result=True) -%}\n {% if information_schema.is_hive_metastore() %}\n {%- for relation in relations -%}\n select '{{ relation.schema }}' as schema,\n '{{ relation.identifier }}' as identifier,\n max(timestamp) as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from (describe history {{ relation.schema|lower }}.{{ relation.identifier|lower }})\n {% if not loop.last %}\n union all\n {% endif %}\n {%- endfor -%}\n {% else %}\n select table_schema as schema,\n table_name as identifier,\n last_altered as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}' and\n (\n {%- for relation in relations -%}\n (table_schema = '{{ relation.schema|lower }}' and\n table_name = '{{ relation.identifier|lower }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n {% endif %}\n {%- endcall -%}\n\n {{ return(load_result('last_modified')) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6481018, "supported_languages": null}, "macro.dbt_databricks.get_view_description": {"name": "get_view_description", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_view_description", "macro_sql": "{% macro get_view_description(relation) %}\n {% call statement('get_view_description', fetch_result=True) -%}\n select *\n from `system`.`information_schema`.`views`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n {%- endcall -%}\n\n {% do return(load_result('get_view_description').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648402, "supported_languages": null}, "macro.dbt_databricks.get_uc_tables": {"name": "get_uc_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_uc_tables", "macro_sql": "{% macro get_uc_tables(relation) %}\n {% call statement('get_uc_tables', fetch_result=True) -%}\n select\n table_name,\n if(table_type in ('EXTERNAL', 'MANAGED', 'MANAGED_SHALLOW_CLONE', 'EXTERNAL_SHALLOW_CLONE'), 'table', lower(table_type)) as table_type,\n lower(data_source_format) as file_format,\n table_owner\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n {% if relation.identifier %}\n and table_name = '{{ relation.identifier|lower }}'\n {% endif %}\n {% endcall %}\n\n {% do return(load_result('get_uc_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648753, "supported_languages": null}, "macro.dbt_spark.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() %}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663456, "supported_languages": null}, "macro.dbt_spark.spark__tblproperties_clause": {"name": "spark__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__tblproperties_clause", "macro_sql": "{% macro spark__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663767, "supported_languages": null}, "macro.dbt_spark.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {{ return(adapter.dispatch('file_format_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__file_format_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663905, "supported_languages": null}, "macro.dbt_spark.spark__file_format_clause": {"name": "spark__file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__file_format_clause", "macro_sql": "{% macro spark__file_format_clause() %}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n {%- if file_format is not none %}\n using {{ file_format }}\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6641312, "supported_languages": null}, "macro.dbt_spark.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.location_clause", "macro_sql": "{% macro location_clause() %}\n {{ return(adapter.dispatch('location_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__location_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664276, "supported_languages": null}, "macro.dbt_spark.spark__location_clause": {"name": "spark__location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__location_clause", "macro_sql": "{% macro spark__location_clause() %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664563, "supported_languages": null}, "macro.dbt_spark.options_clause": {"name": "options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.options_clause", "macro_sql": "{% macro options_clause() -%}\n {{ return(adapter.dispatch('options_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__options_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6646988, "supported_languages": null}, "macro.dbt_spark.spark__options_clause": {"name": "spark__options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__options_clause", "macro_sql": "{% macro spark__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6656451, "supported_languages": null}, "macro.dbt_spark.comment_clause": {"name": "comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.comment_clause", "macro_sql": "{% macro comment_clause() %}\n {{ return(adapter.dispatch('comment_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__comment_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.665798, "supported_languages": null}, "macro.dbt_spark.spark__comment_clause": {"name": "spark__comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__comment_clause", "macro_sql": "{% macro spark__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endif %}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666194, "supported_languages": null}, "macro.dbt_spark.partition_cols": {"name": "partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.partition_cols", "macro_sql": "{% macro partition_cols(label, required=false) %}\n {{ return(adapter.dispatch('partition_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666384, "supported_languages": null}, "macro.dbt_spark.spark__partition_cols": {"name": "spark__partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__partition_cols", "macro_sql": "{% macro spark__partition_cols(label, required=false) %}\n {%- set cols = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666821, "supported_languages": null}, "macro.dbt_spark.clustered_cols": {"name": "clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.clustered_cols", "macro_sql": "{% macro clustered_cols(label, required=false) %}\n {{ return(adapter.dispatch('clustered_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__clustered_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667007, "supported_languages": null}, "macro.dbt_spark.spark__clustered_cols": {"name": "spark__clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__clustered_cols", "macro_sql": "{% macro spark__clustered_cols(label, required=false) %}\n {%- set cols = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n {%- if (cols is not none) and (buckets is not none) %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n ) into {{ buckets }} buckets\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6675909, "supported_languages": null}, "macro.dbt_spark.fetch_tbl_properties": {"name": "fetch_tbl_properties", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.fetch_tbl_properties", "macro_sql": "{% macro fetch_tbl_properties(relation) -%}\n {% call statement('list_properties', fetch_result=True) -%}\n SHOW TBLPROPERTIES {{ relation }}\n {% endcall %}\n {% do return(load_result('list_properties').table) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667808, "supported_languages": null}, "macro.dbt_spark.create_temporary_view": {"name": "create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.create_temporary_view", "macro_sql": "{% macro create_temporary_view(relation, compiled_code) -%}\n {{ return(adapter.dispatch('create_temporary_view', 'dbt')(relation, compiled_code)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__create_temporary_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667983, "supported_languages": null}, "macro.dbt_spark.spark__create_temporary_view": {"name": "spark__create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_temporary_view", "macro_sql": "{% macro spark__create_temporary_view(relation, compiled_code) -%}\n create or replace temporary view {{ relation }} as\n {{ compiled_code }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668092, "supported_languages": null}, "macro.dbt_spark.spark__create_table_as": {"name": "spark__create_table_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_table_as", "macro_sql": "{%- macro spark__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'iceberg'] %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ tblproperties_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_spark.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.tblproperties_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_spark.py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668963, "supported_languages": null}, "macro.dbt_spark.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669142, "supported_languages": null}, "macro.dbt_spark.spark__persist_constraints": {"name": "spark__persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_constraints", "macro_sql": "{% macro spark__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config.enforced and config.get('file_format', 'delta') == 'delta' %}\n {% do alter_table_add_constraints(relation, model.constraints) %}\n {% do alter_column_set_constraints(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.alter_table_add_constraints", "macro.dbt_spark.alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669481, "supported_languages": null}, "macro.dbt_spark.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669651, "supported_languages": null}, "macro.dbt_spark.spark__alter_table_add_constraints": {"name": "spark__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_table_add_constraints", "macro_sql": "{% macro spark__alter_table_add_constraints(relation, constraints) %}\n {% for constraint in constraints %}\n {% if constraint.type == 'check' and not is_incremental() %}\n {%- set constraint_hash = local_md5(column_name ~ \";\" ~ constraint.expression ~ \";\" ~ loop.index) -%}\n {% call statement() %}\n alter table {{ relation }} add constraint {{ constraint.name if constraint.name else constraint_hash }} check ({{ constraint.expression }});\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670092, "supported_languages": null}, "macro.dbt_spark.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67027, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_set_constraints": {"name": "spark__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_set_constraints", "macro_sql": "{% macro spark__alter_column_set_constraints(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set constraints = column_dict[column_name]['constraints'] %}\n {% for constraint in constraints %}\n {% if constraint.type != 'not_null' %}\n {{ exceptions.warn('Invalid constraint for column ' ~ column_name ~ '. Only `not_null` is supported.') }}\n {% else %}\n {% set quoted_name = adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name %}\n {% call statement() %}\n alter table {{ relation }} change column {{ quoted_name }} set not null {{ constraint.expression or \"\" }};\n {% endcall %}\n {% endif %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670861, "supported_languages": null}, "macro.dbt_spark.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671293, "supported_languages": null}, "macro.dbt_spark.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671527, "supported_languages": null}, "macro.dbt_spark.spark__create_view_as": {"name": "spark__create_view_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_view_as", "macro_sql": "{% macro spark__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_spark.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6719759, "supported_languages": null}, "macro.dbt_spark.spark__create_schema": {"name": "spark__create_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_schema", "macro_sql": "{% macro spark__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{relation}}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6721191, "supported_languages": null}, "macro.dbt_spark.spark__drop_schema": {"name": "spark__drop_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_schema", "macro_sql": "{% macro spark__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672259, "supported_languages": null}, "macro.dbt_spark.get_columns_in_relation_raw": {"name": "get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_columns_in_relation_raw", "macro_sql": "{% macro get_columns_in_relation_raw(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation_raw', 'dbt')(relation)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation_raw"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672405, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation_raw": {"name": "spark__get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation_raw", "macro_sql": "{% macro spark__get_columns_in_relation_raw(relation) -%}\n {% call statement('get_columns_in_relation_raw', fetch_result=True) %}\n describe extended {{ relation }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation_raw').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672622, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation": {"name": "spark__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation", "macro_sql": "{% macro spark__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n describe extended {{ relation.include(schema=(schema is not none)) }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672888, "supported_languages": null}, "macro.dbt_spark.spark__list_relations_without_caching": {"name": "spark__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_relations_without_caching", "macro_sql": "{% macro spark__list_relations_without_caching(relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n show table extended in {{ relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673117, "supported_languages": null}, "macro.dbt_spark.list_relations_show_tables_without_caching": {"name": "list_relations_show_tables_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.list_relations_show_tables_without_caching", "macro_sql": "{% macro list_relations_show_tables_without_caching(schema_relation) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('list_relations_without_caching_show_tables', fetch_result=True) -%}\n show tables in {{ schema_relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching_show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6733599, "supported_languages": null}, "macro.dbt_spark.describe_table_extended_without_caching": {"name": "describe_table_extended_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.describe_table_extended_without_caching", "macro_sql": "{% macro describe_table_extended_without_caching(table_name) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('describe_table_extended_without_caching', fetch_result=True) -%}\n describe extended {{ table_name }}\n {% endcall %}\n {% do return(load_result('describe_table_extended_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6736, "supported_languages": null}, "macro.dbt_spark.spark__list_schemas": {"name": "spark__list_schemas", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_schemas", "macro_sql": "{% macro spark__list_schemas(database) -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n show databases\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673813, "supported_languages": null}, "macro.dbt_spark.spark__rename_relation": {"name": "spark__rename_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__rename_relation", "macro_sql": "{% macro spark__rename_relation(from_relation, to_relation) -%}\n {% call statement('rename_relation') -%}\n {% if not from_relation.type %}\n {% do exceptions.raise_database_error(\"Cannot rename a relation with a blank type: \" ~ from_relation.identifier) %}\n {% elif from_relation.type in ('table') %}\n alter table {{ from_relation }} rename to {{ to_relation }}\n {% elif from_relation.type == 'view' %}\n alter view {{ from_relation }} rename to {{ to_relation }}\n {% else %}\n {% do exceptions.raise_database_error(\"Unknown type '\" ~ from_relation.type ~ \"' for relation: \" ~ from_relation.identifier) %}\n {% endif %}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674293, "supported_languages": null}, "macro.dbt_spark.spark__drop_relation": {"name": "spark__drop_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_relation", "macro_sql": "{% macro spark__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674477, "supported_languages": null}, "macro.dbt_spark.spark__generate_database_name": {"name": "spark__generate_database_name", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__generate_database_name", "macro_sql": "{% macro spark__generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(None) %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674613, "supported_languages": null}, "macro.dbt_spark.spark__persist_docs": {"name": "spark__persist_docs", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_docs", "macro_sql": "{% macro spark__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do alter_column_comment(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674838, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_comment": {"name": "spark__alter_column_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_comment", "macro_sql": "{% macro spark__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'hudi', 'iceberg'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n {% if relation.is_iceberg %}\n alter table {{ relation }} alter column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% else %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endif %}\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.675686, "supported_languages": null}, "macro.dbt_spark.spark__make_temp_relation": {"name": "spark__make_temp_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__make_temp_relation", "macro_sql": "{% macro spark__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(path = {\n \"identifier\": tmp_identifier\n }) -%}\n\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6760252, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_type": {"name": "spark__alter_column_type", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_type", "macro_sql": "{% macro spark__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ column_name }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67621, "supported_languages": null}, "macro.dbt_spark.spark__alter_relation_add_remove_columns": {"name": "spark__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_relation_add_remove_columns", "macro_sql": "{% macro spark__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if remove_columns %}\n {% if relation.is_delta %}\n {% set platform_name = 'Delta Lake' %}\n {% elif relation.is_iceberg %}\n {% set platform_name = 'Iceberg' %}\n {% else %}\n {% set platform_name = 'Apache Spark' %}\n {% endif %}\n {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}\n {% endif %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% if add_columns %} add columns {% endif %}\n {% for column in add_columns %}\n {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.676926, "supported_languages": null}, "macro.dbt_spark.spark__copy_grants": {"name": "spark__copy_grants", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__copy_grants", "macro_sql": "{% macro spark__copy_grants() %}\n\n {% if config.materialized == 'view' %}\n {#-- Spark views don't copy grants when they're replaced --#}\n {{ return(False) }}\n\n {% else %}\n {#-- This depends on how we're replacing the table, which depends on its file format\n -- Just play it safe by assuming that grants have been copied over, and need to be checked / possibly revoked\n -- We can make this more efficient in the future\n #}\n {{ return(True) }}\n\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677521, "supported_languages": null}, "macro.dbt_spark.spark__get_grant_sql": {"name": "spark__get_grant_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_grant_sql", "macro_sql": "\n\n\n{%- macro spark__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6777, "supported_languages": null}, "macro.dbt_spark.spark__get_revoke_sql": {"name": "spark__get_revoke_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_revoke_sql", "macro_sql": "\n\n\n{%- macro spark__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677873, "supported_languages": null}, "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement": {"name": "spark__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n\n{%- macro spark__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(False) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677966, "supported_languages": null}, "macro.dbt_spark.spark__call_dcl_statements": {"name": "spark__call_dcl_statements", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__call_dcl_statements", "macro_sql": "{% macro spark__call_dcl_statements(dcl_statement_list) %}\n {% for dcl_statement in dcl_statement_list %}\n {% call statement('grant_or_revoke') %}\n {{ dcl_statement }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.678161, "supported_languages": null}, "macro.dbt_spark.spark__get_binding_char": {"name": "spark__get_binding_char", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__get_binding_char", "macro_sql": "{% macro spark__get_binding_char() %}\n {{ return('?' if target.method == 'odbc' else '%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.679692, "supported_languages": null}, "macro.dbt_spark.spark__reset_csv_table": {"name": "spark__reset_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__reset_csv_table", "macro_sql": "{% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {{ adapter.drop_relation(old_relation) }}\n {% endif %}\n {% set sql = create_csv_table(model, agate_table) %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6799822, "supported_languages": null}, "macro.dbt_spark.spark__load_csv_rows": {"name": "spark__load_csv_rows", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__load_csv_rows", "macro_sql": "{% macro spark__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.681087, "supported_languages": null}, "macro.dbt_spark.spark__create_csv_table": {"name": "spark__create_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__create_csv_table", "macro_sql": "{% macro spark__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6820521, "supported_languages": null}, "macro.dbt_spark.materialization_view_spark": {"name": "materialization_view_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_spark.materialization_view_spark", "macro_sql": "{% materialization view, adapter='spark' -%}\n {{ return(create_or_replace_view()) }}\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.create_or_replace_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.682219, "supported_languages": ["sql"]}, "macro.dbt_spark.materialization_table_spark": {"name": "materialization_table_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.materialization_table_spark", "macro_sql": "{% materialization table, adapter = 'spark', supported_languages=['sql', 'python'] %}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation is not none %}\n {% set is_delta = (old_relation.is_delta and config.get('file_format', validator=validation.any[basestring]) == 'delta') %}\n {% set is_iceberg = (old_relation.is_iceberg and config.get('file_format', validator=validation.any[basestring]) == 'iceberg') %}\n {% set old_relation_type = old_relation.type %}\n {% else %}\n {% set is_delta = false %}\n {% set is_iceberg = false %}\n {% set old_relation_type = target_relation.type %}\n {% endif %}\n\n {% if not is_delta and not is_iceberg %}\n {% set existing_relation = target_relation %}\n {{ adapter.drop_relation(existing_relation.incorporate(type=old_relation_type)) }}\n {% endif %}\n\n -- build model\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt_spark.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6857522, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.py_write_table": {"name": "py_write_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_write_table", "macro_sql": "{% macro py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\n# make sure pyspark exists in the namepace, for 7.3.x-scala2.12 it does not exist\nimport pyspark\n# make sure pandas exists before using it\ntry:\n import pandas\n pandas_available = True\nexcept ImportError:\n pandas_available = False\n\n# make sure pyspark.pandas exists before using it\ntry:\n import pyspark.pandas\n pyspark_pandas_api_available = True\nexcept ImportError:\n pyspark_pandas_api_available = False\n\n# make sure databricks.koalas exists before using it\ntry:\n import databricks.koalas\n koalas_available = True\nexcept ImportError:\n koalas_available = False\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\nif pyspark_pandas_api_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = pyspark.pandas.frame.DataFrame(df)\nelif koalas_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\ndf.write.mode(\"overwrite\").format(\"{{ config.get('file_format', 'delta') }}\").option(\"overwriteSchema\", \"true\").saveAsTable(\"{{ target_relation }}\")\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686034, "supported_languages": null}, "macro.dbt_spark.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n# how to execute python model in notebook\n# dbt = dbtObj(spark.table)\n# df = model(dbt, spark)\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686104, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_hash_arguments": {"name": "spark__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_hash_arguments", "macro_sql": "{% macro spark__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as string ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.692856, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_string_as_time": {"name": "spark__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_string_as_time", "macro_sql": "{% macro spark__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"to_timestamp('\" ~ timestamp ~ \"')\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693005, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_merge_sql": {"name": "spark__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_merge_sql", "macro_sql": "{% macro spark__snapshot_merge_sql(target, source, insert_cols) -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {% if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {% else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {% endif %}\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert *\n ;\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693233, "supported_languages": null}, "macro.dbt_spark.spark_build_snapshot_staging_table": {"name": "spark_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark_build_snapshot_staging_table", "macro_sql": "{% macro spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {% if target_relation.is_iceberg %}\n {# iceberg catalog does not support create view, but regular spark does. We removed the catalog and schema #}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=none,\n database=none,\n type='view') -%}\n {% else %}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=none,\n type='view') -%}\n {% endif %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_view_as(tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693888, "supported_languages": null}, "macro.dbt_spark.spark__post_snapshot": {"name": "spark__post_snapshot", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__post_snapshot", "macro_sql": "{% macro spark__post_snapshot(staging_relation) %}\n {% do adapter.drop_relation(staging_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.694009, "supported_languages": null}, "macro.dbt_spark.spark__create_columns": {"name": "spark__create_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__create_columns", "macro_sql": "{% macro spark__create_columns(relation, columns) %}\n {% if columns|length > 0 %}\n {% call statement() %}\n alter table {{ relation }} add columns (\n {% for column in columns %}\n `{{ column.name }}` {{ column.data_type }} {{- ',' if not loop.last -}}\n {% endfor %}\n );\n {% endcall %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6943328, "supported_languages": null}, "macro.dbt_spark.materialization_snapshot_spark": {"name": "materialization_snapshot_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.materialization_snapshot_spark", "macro_sql": "{% materialization snapshot, adapter='spark' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'parquet') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=none,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if file_format not in ['delta', 'iceberg', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_iceberg and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.schema) %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.create_schema", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6975749, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__can_clone_table": {"name": "spark__can_clone_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__can_clone_table", "macro_sql": "{% macro spark__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6994078, "supported_languages": null}, "macro.dbt_spark.spark__create_or_replace_clone": {"name": "spark__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__create_or_replace_clone", "macro_sql": "{% macro spark__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.699519, "supported_languages": null}, "macro.dbt_spark.materialization_clone_spark": {"name": "materialization_clone_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.materialization_clone_spark", "macro_sql": "\n\n{%- materialization clone, adapter='spark' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if file_format != 'delta' -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n shallow clone requires file_format be set to 'delta'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {%- elif other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.70128, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__get_merge_update_columns": {"name": "spark__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/column_helpers.sql", "original_file_path": "macros/materializations/incremental/column_helpers.sql", "unique_id": "macro.dbt_spark.spark__get_merge_update_columns", "macro_sql": "{% macro spark__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.702069, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_file_format": {"name": "dbt_spark_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_file_format", "macro_sql": "{% macro dbt_spark_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'iceberg', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703007, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy": {"name": "dbt_spark_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'append', 'merge', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_merge_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via endpoint\n Use the 'append' or 'merge' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'iceberg', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_merge_msg) %}\n {% endif %}\n {% if raw_strategy == 'insert_overwrite' and target.endpoint %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_endpoint_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703701, "supported_languages": null}, "macro.dbt_spark.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation, existing_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n {% if existing_relation.is_iceberg %}\n {# removed table from statement for iceberg #}\n insert overwrite {{ target_relation }}\n {# removed partition_cols for iceberg as well #}\n {% else %}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {% endif %}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705375, "supported_languages": null}, "macro.dbt_spark.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert into table {{ target_relation }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705639, "supported_languages": null}, "macro.dbt_spark.spark__get_merge_sql": {"name": "spark__get_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.spark__get_merge_sql", "macro_sql": "{% macro spark__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n when matched then update set\n {% if update_columns -%}{%- for column_name in update_columns %}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {%- else %} * {% endif %}\n\n when not matched then insert *\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.706816, "supported_languages": null}, "macro.dbt_spark.dbt_spark_get_incremental_sql": {"name": "dbt_spark_get_incremental_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro_sql": "{% macro dbt_spark_get_incremental_sql(strategy, source, target, existing, unique_key, incremental_predicates) %}\n {%- if strategy == 'append' -%}\n {#-- insert new records into existing table, without updating or overwriting #}\n {{ get_insert_into_sql(source, target) }}\n {%- elif strategy == 'insert_overwrite' -%}\n {#-- insert statements don't like CTEs, so support them via a temp view #}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'merge' -%}\n {#-- merge all columns for datasources which implement MERGE INTO (e.g. databricks, iceberg) - schema changes are handled for us #}\n {{ get_merge_sql(target, source, unique_key, dest_columns=none, incremental_predicates=incremental_predicates) }}\n {%- else -%}\n {% set no_sql_for_strategy_msg -%}\n No known SQL for the incremental strategy provided: {{ strategy }}\n {%- endset %}\n {%- do exceptions.raise_compiler_error(no_sql_for_strategy_msg) -%}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_insert_into_sql", "macro.dbt_spark.get_insert_overwrite_sql", "macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.707412, "supported_languages": null}, "macro.dbt_spark.materialization_incremental_spark": {"name": "materialization_incremental_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_spark.materialization_incremental_spark", "macro_sql": "{% materialization incremental, adapter='spark', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='parquet') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'append' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set file_format = dbt_spark_validate_get_file_format(raw_file_format) -%}\n {%- set strategy = dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}\n {%- set target_relation = this -%}\n {%- set existing_relation = load_relation(this) -%}\n {%- set tmp_relation = make_temp_relation(this) -%}\n\n {#-- for SQL model we will create temp view that doesn't have database and schema --#}\n {%- if language == 'sql'-%}\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode --#}\n {%- if strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- elif existing_relation.is_view or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- else -%}\n {#-- Relation must be merged --#}\n {%- call statement('create_tmp_relation', language=language) -%}\n {{ create_table_as(True, tmp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, tmp_relation, existing_relation) -%}\n {%- call statement('main') -%}\n {{ dbt_spark_get_incremental_sql(strategy, tmp_relation, target_relation, existing_relation, unique_key, incremental_predicates) }}\n {%- endcall -%}\n {%- if language == 'python' -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why do neither drop_relation or adapter.drop_relation work here?!\n --#}\n {% call statement('drop_relation') -%}\n drop table if exists {{ tmp_relation }}\n {%- endcall %}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_spark.dbt_spark_validate_get_file_format", "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_spark.persist_constraints", "macro.dbt.should_full_refresh", "macro.dbt.process_schema_changes", "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710429, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.spark__concat": {"name": "spark__concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt_spark.spark__concat", "macro_sql": "{% macro spark__concat(fields) -%}\n concat({{ fields|join(', ') }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710591, "supported_languages": null}, "macro.dbt_spark.assert_not_null": {"name": "assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.assert_not_null", "macro_sql": "{% macro assert_not_null(function, arg) -%}\n {{ return(adapter.dispatch('assert_not_null', 'dbt')(function, arg)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710856, "supported_languages": null}, "macro.dbt_spark.spark__assert_not_null": {"name": "spark__assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.spark__assert_not_null", "macro_sql": "{% macro spark__assert_not_null(function, arg) %}\n\n coalesce({{function}}({{arg}}), nvl2({{function}}({{arg}}), assert_true({{function}}({{arg}}) is not null), null))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.711019, "supported_languages": null}, "macro.dbt_spark.spark__dateadd": {"name": "spark__dateadd", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_spark.spark__dateadd", "macro_sql": "{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {%- set clock_component -%}\n {# make sure the dates + timestamps are real, otherwise raise an error asap #}\n to_unix_timestamp({{ assert_not_null('to_timestamp', from_date_or_timestamp) }})\n - to_unix_timestamp({{ assert_not_null('date', from_date_or_timestamp) }})\n {%- endset -%}\n\n {%- if datepart in ['day', 'week'] -%}\n\n {%- set multiplier = 7 if datepart == 'week' else 1 -%}\n\n to_timestamp(\n to_unix_timestamp(\n date_add(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ['month', 'quarter', 'year'] -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'month' -%} 1\n {%- elif datepart == 'quarter' -%} 3\n {%- elif datepart == 'year' -%} 12\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp(\n add_months(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n {{ assert_not_null('to_unix_timestamp', from_date_or_timestamp) }}\n + cast({{interval}} * {{multiplier}} as int)\n )\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro dateadd not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713269, "supported_languages": null}, "macro.dbt_spark.spark__current_timestamp": {"name": "spark__current_timestamp", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/timestamps.sql", "original_file_path": "macros/utils/timestamps.sql", "unique_id": "macro.dbt_spark.spark__current_timestamp", "macro_sql": "{% macro spark__current_timestamp() -%}\n current_timestamp()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713365, "supported_languages": null}, "macro.dbt_spark.spark__escape_single_quotes": {"name": "spark__escape_single_quotes", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt_spark.spark__escape_single_quotes", "macro_sql": "{% macro spark__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"\\\\'\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713526, "supported_languages": null}, "macro.dbt_spark.spark__listagg": {"name": "spark__listagg", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_spark.spark__listagg", "macro_sql": "{% macro spark__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if order_by_clause %}\n {{ exceptions.warn(\"order_by_clause is not supported for listagg on Spark/Databricks\") }}\n {% endif %}\n\n {% set collect_list %} collect_list({{ measure }}) {% endset %}\n\n {% set limited %} slice({{ collect_list }}, 1, {{ limit_num }}) {% endset %}\n\n {% set collected = limited if limit_num else collect_list %}\n\n {% set final %} array_join({{ collected }}, {{ delimiter_text }}) {% endset %}\n\n {% do return(final) %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.714139, "supported_languages": null}, "macro.dbt_spark.spark__datediff": {"name": "spark__datediff", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_spark.spark__datediff", "macro_sql": "{% macro spark__datediff(first_date, second_date, datepart) %}\n\n {%- if datepart in ['day', 'week', 'month', 'quarter', 'year'] -%}\n\n {# make sure the dates are real, otherwise raise an error asap #}\n {% set first_date = assert_not_null('date', first_date) %}\n {% set second_date = assert_not_null('date', second_date) %}\n\n {%- endif -%}\n\n {%- if datepart == 'day' -%}\n\n datediff({{second_date}}, {{first_date}})\n\n {%- elif datepart == 'week' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(datediff({{second_date}}, {{first_date}})/7)\n else ceil(datediff({{second_date}}, {{first_date}})/7)\n end\n\n -- did we cross a week boundary (Sunday)?\n + case\n when {{first_date}} < {{second_date}} and dayofweek({{second_date}}) < dayofweek({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofweek({{second_date}}) > dayofweek({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'month' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}})))\n else ceil(months_between(date({{second_date}}), date({{first_date}})))\n end\n\n -- did we cross a month boundary?\n + case\n when {{first_date}} < {{second_date}} and dayofmonth({{second_date}}) < dayofmonth({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofmonth({{second_date}}) > dayofmonth({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'quarter' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}}))/3)\n else ceil(months_between(date({{second_date}}), date({{first_date}}))/3)\n end\n\n -- did we cross a quarter boundary?\n + case\n when {{first_date}} < {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n < (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then 1\n when {{first_date}} > {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n > (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then -1\n else 0 end\n\n {%- elif datepart == 'year' -%}\n\n year({{second_date}}) - year({{first_date}})\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set divisor -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n case when {{first_date}} < {{second_date}}\n then ceil((\n {# make sure the timestamps are real, otherwise raise an error asap #}\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n else floor((\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n end\n\n {% if datepart == 'millisecond' %}\n + cast(date_format({{second_date}}, 'SSS') as int)\n - cast(date_format({{first_date}}, 'SSS') as int)\n {% endif %}\n\n {% if datepart == 'microsecond' %}\n {% set capture_str = '[0-9]{4}-[0-9]{2}-[0-9]{2}.[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]{6})' %}\n -- Spark doesn't really support microseconds, so this is a massive hack!\n -- It will only work if the timestamp-string is of the format\n -- 'yyyy-MM-dd-HH mm.ss.SSSSSS'\n + cast(regexp_extract({{second_date}}, '{{capture_str}}', 1) as int)\n - cast(regexp_extract({{first_date}}, '{{capture_str}}', 1) as int)\n {% endif %}\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro datediff not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7200809, "supported_languages": null}, "macro.dbt_spark.spark__safe_cast": {"name": "spark__safe_cast", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.spark__safe_cast", "macro_sql": "{% macro spark__safe_cast(field, type) %}\n{%- set field_clean = field.strip('\"').strip(\"'\") if (cast_from_string_unsupported_for(type) and field is string) else field -%}\ncast({{field_clean}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.cast_from_string_unsupported_for"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7204401, "supported_languages": null}, "macro.dbt_spark.cast_from_string_unsupported_for": {"name": "cast_from_string_unsupported_for", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.cast_from_string_unsupported_for", "macro_sql": "{% macro cast_from_string_unsupported_for(type) %}\n {{ return(type.lower().startswith('struct') or type.lower().startswith('array') or type.lower().startswith('map')) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720673, "supported_languages": null}, "macro.dbt_spark.spark__any_value": {"name": "spark__any_value", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_spark.spark__any_value", "macro_sql": "{% macro spark__any_value(expression) -%}\n {#-- return any value (non-deterministic) --#}\n first({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720806, "supported_languages": null}, "macro.dbt_spark.spark__array_concat": {"name": "spark__array_concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt_spark.spark__array_concat", "macro_sql": "{% macro spark__array_concat(array_1, array_2) -%}\n concat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720957, "supported_languages": null}, "macro.dbt_spark.spark__bool_or": {"name": "spark__bool_or", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt_spark.spark__bool_or", "macro_sql": "{% macro spark__bool_or(expression) -%}\n\n max({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.721088, "supported_languages": null}, "macro.dbt_spark.spark__split_part": {"name": "spark__split_part", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_spark.spark__split_part", "macro_sql": "{% macro spark__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% if part_number >= 0 %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[({{ part_number - 1 if part_number > 0 else part_number }})]\n\n {% endset %}\n\n {% else %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[(\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1 + {{ part_number }}\n )]\n\n {% endset %}\n\n {% endif %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722142, "supported_languages": null}, "macro.dbt_spark.spark__array_construct": {"name": "spark__array_construct", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt_spark.spark__array_construct", "macro_sql": "{% macro spark__array_construct(inputs, data_type) -%}\n array( {{ inputs|join(' , ') }} )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722311, "supported_languages": null}, "macro.dbt_spark.spark__array_append": {"name": "spark__array_append", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt_spark.spark__array_append", "macro_sql": "{% macro spark__array_append(array, new_element) -%}\n {{ array_concat(array, array_construct([new_element])) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.array_concat", "macro.dbt.array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722492, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72342, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723584, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723713, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723829, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723947, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724263, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72451, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724753, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7252579, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.725742, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7284422, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7285912, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.728775, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7294762, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7296178, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.729759, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7308888, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7321742, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.73771, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.737952, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738092, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738164, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738283, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7383811, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7385468, "supported_languages": null}, "macro.dbt.get_snapshot_table_column_names": {"name": "get_snapshot_table_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_snapshot_table_column_names", "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at'}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7387378, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where {{ columns.dbt_valid_to }} is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.739788, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7399502, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7403069, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.740653, "supported_languages": null}, "macro.dbt.get_updated_at_column_data_type": {"name": "get_updated_at_column_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_updated_at_column_data_type", "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741133, "supported_languages": null}, "macro.dbt.check_time_data_types": {"name": "check_time_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.check_time_data_types", "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_updated_at_column_data_type", "macro.dbt.get_snapshot_get_time_data_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741546, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.valid_snapshot_target(target_relation, columns) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.746002, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7482162, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749175, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7494242, "supported_languages": null}, "macro.dbt.get_unit_test_sql": {"name": "get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_unit_test_sql", "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_unit_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749598, "supported_languages": null}, "macro.dbt.default__get_unit_test_sql": {"name": "default__get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_unit_test_sql", "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7500942, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7504158, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7507288, "supported_languages": null}, "macro.dbt.materialization_unit_default": {"name": "materialization_unit_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/unit.sql", "original_file_path": "macros/materializations/tests/unit.sql", "unique_id": "macro.dbt.materialization_unit_default", "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_empty_subquery_sql", "macro.dbt.get_expected_sql", "macro.dbt.get_unit_test_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7521741, "supported_languages": ["sql"]}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756201, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756525, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756736, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.757851, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.75805, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7585862, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.760961, "supported_languages": ["sql"]}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.763325, "supported_languages": ["sql"]}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7646668, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765106, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.76564, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765835, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7664142, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.771425, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.772757, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.77298, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.773788, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774013, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774538, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775049, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775977, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776169, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776327, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7765691, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776725, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7769668, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7771251, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777345, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777499, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777623, "supported_languages": null}, "macro.dbt.get_incremental_microbatch_sql": {"name": "get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_microbatch_sql", "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_microbatch_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777774, "supported_languages": null}, "macro.dbt.default__get_incremental_microbatch_sql": {"name": "default__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_microbatch_sql", "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7779121, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7781398, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.782465, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7869802, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.787961, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.788921, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789614, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789815, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7899132, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790163, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790299, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.793341, "supported_languages": ["sql"]}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.796114, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80035, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80112, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8013299, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801737, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801903, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80202, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8021429, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802243, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8023849, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802489, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8028958, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.803055, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8041549, "supported_languages": null}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804568, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804885, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8053389, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8055522, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8057861, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806116, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806323, "supported_languages": null}, "macro.dbt.get_drop_sql": {"name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806918, "supported_languages": null}, "macro.dbt.default__get_drop_sql": {"name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807232, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8073862, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807553, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807711, "supported_languages": null}, "macro.dbt.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.808418, "supported_languages": null}, "macro.dbt.default__get_replace_sql": {"name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80947, "supported_languages": null}, "macro.dbt.get_create_intermediate_sql": {"name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.809802, "supported_languages": null}, "macro.dbt.default__get_create_intermediate_sql": {"name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810016, "supported_languages": null}, "macro.dbt.drop_schema_named": {"name": "drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.drop_schema_named", "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_schema_named"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81025, "supported_languages": null}, "macro.dbt.default__drop_schema_named": {"name": "default__drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.default__drop_schema_named", "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810434, "supported_languages": null}, "macro.dbt.get_drop_backup_sql": {"name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81071, "supported_languages": null}, "macro.dbt.default__get_drop_backup_sql": {"name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810894, "supported_languages": null}, "macro.dbt.get_rename_sql": {"name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811506, "supported_languages": null}, "macro.dbt.default__get_rename_sql": {"name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811878, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812062, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812324, "supported_languages": null}, "macro.dbt.get_create_backup_sql": {"name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812639, "supported_languages": null}, "macro.dbt.default__get_create_backup_sql": {"name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812882, "supported_languages": null}, "macro.dbt.get_create_sql": {"name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8133051, "supported_languages": null}, "macro.dbt.default__get_create_sql": {"name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813689, "supported_languages": null}, "macro.dbt.get_rename_intermediate_sql": {"name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813972, "supported_languages": null}, "macro.dbt.default__get_rename_intermediate_sql": {"name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8141482, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8143559, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814456, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_sql": {"name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8146849, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_sql": {"name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814875, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815148, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815263, "supported_languages": null}, "macro.dbt.get_rename_materialized_view_sql": {"name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8154979, "supported_languages": null}, "macro.dbt.default__get_rename_materialized_view_sql": {"name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8156219, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8161569, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816313, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8165581, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816685, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816923, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817044, "supported_languages": null}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817939, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818042, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8184981, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818639, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818753, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.819914, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820245, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8205361, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820756, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82086, "supported_languages": null}, "macro.dbt.get_replace_table_sql": {"name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8210938, "supported_languages": null}, "macro.dbt.default__get_replace_table_sql": {"name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8212202, "supported_languages": null}, "macro.dbt.get_rename_table_sql": {"name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82146, "supported_languages": null}, "macro.dbt.default__get_rename_table_sql": {"name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82159, "supported_languages": null}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.822352, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8225112, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8228748, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823435, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8238149, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823968, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824113, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824321, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824422, "supported_languages": null}, "macro.dbt.get_replace_view_sql": {"name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.825161, "supported_languages": null}, "macro.dbt.default__get_replace_view_sql": {"name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82529, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82631, "supported_languages": null}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8264818, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8266811, "supported_languages": null}, "macro.dbt.get_rename_view_sql": {"name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.826918, "supported_languages": null}, "macro.dbt.default__get_rename_view_sql": {"name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8270478, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8274088, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8275461, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.827704, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828066, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828371, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828622, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828834, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.829317, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.83059, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8313391, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8315961, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.833154, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.834102, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8347142, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8349109, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835105, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835169, "supported_languages": null}, "macro.dbt.get_intervals_between": {"name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835808, "supported_languages": null}, "macro.dbt.default__get_intervals_between": {"name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8363059, "supported_languages": null}, "macro.dbt.date_spine": {"name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836503, "supported_languages": null}, "macro.dbt.default__date_spine": {"name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836822, "supported_languages": null}, "macro.dbt.date": {"name": "date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.date", "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837112, "supported_languages": null}, "macro.dbt.default__date": {"name": "default__date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.default__date", "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837353, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837631, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837771, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837983, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838089, "supported_languages": null}, "macro.dbt.get_powers_of_two": {"name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838809, "supported_languages": null}, "macro.dbt.default__get_powers_of_two": {"name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839253, "supported_languages": null}, "macro.dbt.generate_series": {"name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839426, "supported_languages": null}, "macro.dbt.default__generate_series": {"name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839862, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840083, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8401742, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8404648, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840607, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840808, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840875, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841106, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841234, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84149, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8416152, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842151, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842484, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842758, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842896, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843133, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843246, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843461, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843594, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8438652, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8439958, "supported_languages": null}, "macro.dbt.cast": {"name": "cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.cast", "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844224, "supported_languages": null}, "macro.dbt.default__cast": {"name": "default__cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.default__cast", "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844336, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8445451, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844633, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8448782, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844993, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845202, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845294, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846126, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846256, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846394, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8465211, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846658, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8467891, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846924, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8470762, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847214, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847407, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8475451, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8476732, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8478148, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8479412, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848181, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848293, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485062, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485959, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848897, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84913, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.849261, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8497112, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8498502, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8500328, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850267, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.85038, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850707, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850914, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8511622, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851361, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851688, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851847, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851985, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852136, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852708, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852832, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852951, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853037, "supported_languages": null}, "macro.dbt.get_snapshot_get_time_data_type": {"name": "get_snapshot_get_time_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.get_snapshot_get_time_data_type", "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.default__snapshot_get_time", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8533702, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853508, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853577, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853718, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853857, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854583, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854707, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8548422, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8552508, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855412, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855531, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8556669, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855777, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857485, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857631, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857822, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858082, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858318, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8586, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858756, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858906, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859114, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859583, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859786, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859914, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.860286, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86064, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8608792, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8611462, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862612, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862713, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8628511, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862943, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863224, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863383, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863481, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863672, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863852, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86405, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.864309, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8645408, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865141, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865317, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865537, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865737, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8668082, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8671901, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_limit_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8673549, "supported_languages": null}, "macro.dbt.default__get_limit_sql": {"name": "default__get_limit_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_sql", "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ compiled_code }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86752, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86812, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868274, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868457, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8686101, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86885, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8692548, "supported_languages": null}, "macro.dbt.get_catalog_relations": {"name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872187, "supported_languages": null}, "macro.dbt.default__get_catalog_relations": {"name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872406, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872573, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8727858, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872941, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8730712, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8732271, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87344, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8736901, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8739462, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874113, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874256, "supported_languages": null}, "macro.dbt.get_catalog_for_single_relation": {"name": "get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_for_single_relation", "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_catalog_for_single_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874414, "supported_languages": null}, "macro.dbt.default__get_catalog_for_single_relation": {"name": "default__get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_for_single_relation", "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8745549, "supported_languages": null}, "macro.dbt.get_relations": {"name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874696, "supported_languages": null}, "macro.dbt.default__get_relations": {"name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8748312, "supported_languages": null}, "macro.dbt.get_relation_last_modified": {"name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_relation_last_modified"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87501, "supported_languages": null}, "macro.dbt.default__get_relation_last_modified": {"name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8751612, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877135, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877271, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877537, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8777258, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877902, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8780658, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879136, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879442, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879605, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879909, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880113, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880679, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8809001, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.88154, "supported_languages": null}, "macro.dbt.get_fixture_sql": {"name": "get_fixture_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_fixture_sql", "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- for column in columns_in_relation -%}\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.safe_cast", "macro.dbt.validate_fixture_rows", "macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.884523, "supported_languages": null}, "macro.dbt.get_expected_sql": {"name": "get_expected_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_expected_sql", "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8849862, "supported_languages": null}, "macro.dbt.format_row": {"name": "format_row", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.format_row", "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.string_literal", "macro.dbt.escape_single_quotes", "macro.dbt.safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8859332, "supported_languages": null}, "macro.dbt.validate_fixture_rows": {"name": "validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.validate_fixture_rows", "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__validate_fixture_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8861172, "supported_languages": null}, "macro.dbt.default__validate_fixture_rows": {"name": "default__validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.default__validate_fixture_rows", "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.886199, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887635, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887774, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.888478, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8888261, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8893209, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8898182, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.889886, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890347, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8905559, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890809, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.891062, "supported_languages": null}}, "docs": {"doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {}, "groups": {}, "selectors": {}, "disabled": {}, "parent_map": {"model.dbt_demo.diamonds_list_colors": ["model.dbt_demo.diamonds_four_cs"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["source.dbt_demo.raw.diamonds"], "model.dbt_demo.zzz_game_details": [], "model.dbt_demo.second_dbt_model": ["model.dbt_demo.first_dbt_model"], "model.dbt_demo.first_dbt_model": [], "test.dbt_demo.zzz_game_details_check_scores": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_game_details_check_dates": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_win_loss_records_check_records": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.diamonds_list_colors_test": ["model.dbt_demo.diamonds_list_colors"], "seed.dbt_demo.country_codes": [], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": ["model.dbt_demo.second_dbt_model"], "model.dbt_demo.changed_country": ["snapshot.dbt_demo.country_snapshot"], "model.dbt_demo.country": ["seed.dbt_demo.country_codes"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.country"], "model.dbt_demo.zzz_win_loss_records": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": ["model.dbt_demo.zzz_win_loss_records"], "source.dbt_demo.raw.diamonds": [], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": ["model.dbt_demo.diamonds_list_colors"]}, "child_map": {"model.dbt_demo.diamonds_list_colors": ["test.dbt_demo.diamonds_list_colors_test", "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["model.dbt_demo.diamonds_list_colors"], "model.dbt_demo.zzz_game_details": ["model.dbt_demo.zzz_win_loss_records", "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "test.dbt_demo.zzz_game_details_check_dates", "test.dbt_demo.zzz_game_details_check_scores"], "model.dbt_demo.second_dbt_model": ["test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4"], "model.dbt_demo.first_dbt_model": ["model.dbt_demo.second_dbt_model", "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "test.dbt_demo.unique_first_dbt_model_id.9dda05743e"], "test.dbt_demo.zzz_game_details_check_scores": [], "test.dbt_demo.zzz_game_details_check_dates": [], "test.dbt_demo.zzz_win_loss_records_check_records": [], "test.dbt_demo.diamonds_list_colors_test": [], "seed.dbt_demo.country_codes": ["model.dbt_demo.country"], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": [], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": [], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": [], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": [], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": [], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": [], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": [], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": [], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": [], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": [], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": [], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": [], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": [], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": [], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": [], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": [], "model.dbt_demo.changed_country": [], "model.dbt_demo.country": ["snapshot.dbt_demo.country_snapshot"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.changed_country"], "model.dbt_demo.zzz_win_loss_records": ["test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "test.dbt_demo.zzz_win_loss_records_check_records"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": [], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": [], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": [], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": [], "source.dbt_demo.raw.diamonds": ["model.dbt_demo.diamonds_four_cs"], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": []}, "group_map": {}, "saved_queries": {}, "semantic_models": {}, "unit_tests": {"unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": {"model": "diamonds_list_colors", "given": [{"input": "ref('diamonds_four_cs')", "rows": [{"carat": 0.23, "cut": "Ideal", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "K", "clarity": "SI2"}], "format": "dict", "fixture": null}], "expect": {"rows": [{"color": "E"}, {"color": "K"}], "format": "dict", "fixture": null}, "name": "test_get_distinct_colors", "resource_type": "unit_test", "package_name": "dbt_demo", "path": "sql_model1/test_get_distinct_colors.yml", "original_file_path": "src/models/sql_model1/test_get_distinct_colors.yml", "unique_id": "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors", "fqn": ["dbt_demo", "models", "sql_model1", "diamonds_list_colors", "test_get_distinct_colors"], "description": "Check a distinct list of colors is returned.", "overrides": null, "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "config": {"tags": [], "meta": {}}, "checksum": "483c13eb849bf29c60173f6b4583def3c4a3b8c20e674a28c0db2c3e81834bb1", "schema": "dbt_dev", "created_at": 1733486794.4994261, "versions": null, "version": null}}} \ No newline at end of file diff --git a/contrib/dbt_factory/tests/test_dbt_factory.py b/contrib/dbt_factory/tests/test_dbt_factory.py new file mode 100644 index 00000000..8709ad20 --- /dev/null +++ b/contrib/dbt_factory/tests/test_dbt_factory.py @@ -0,0 +1,440 @@ +import json +from pathlib import Path + +import pytest + +from databricks_dbt_factory.utils import read_dbt_manifest + +BASE_PATH = str(Path(__file__).resolve().parent) + + +def _model(package: str, name: str, depends_on: list[str] | None = None) -> tuple[str, dict]: + full_name = f"model.{package}.{name}" + return full_name, { + "resource_type": "model", + "name": name, + "package_name": package, + "fqn": [package, name], + "depends_on": {"nodes": depends_on or []}, + } + + +def _test( + package: str, name: str, depends_on: list[str], severity: str = "error", test_hash: str = "" +) -> tuple[str, dict]: + full_name = f"test.{package}.{name}" + (f".{test_hash}" if test_hash else "") + return full_name, { + "resource_type": "test", + "name": name, + "package_name": package, + "fqn": [package, name], + "depends_on": {"nodes": depends_on}, + "config": {"severity": severity}, + } + + +def _seed(package: str, name: str) -> tuple[str, dict]: + full_name = f"seed.{package}.{name}" + return full_name, { + "resource_type": "seed", + "name": name, + "package_name": package, + "fqn": [package, name], + "depends_on": {"nodes": []}, + } + + +def _snapshot(package: str, name: str, depends_on: list[str] | None = None) -> tuple[str, dict]: + full_name = f"snapshot.{package}.{name}" + return full_name, { + "resource_type": "snapshot", + "name": name, + "package_name": package, + "fqn": [package, name], + "depends_on": {"nodes": depends_on or []}, + } + + +def _source(package: str, source_name: str, table: str) -> tuple[str, dict]: + full_name = f"source.{package}.{source_name}.{table}" + return full_name, { + "resource_type": "source", + "name": table, + "source_name": source_name, + "package_name": package, + "fqn": [package, source_name, table], + } + + +def _commands(task: dict) -> list[str]: + """Extracts the dbt commands from a rendered (serverless notebook) task dict.""" + return json.loads(task["notebook_task"]["base_parameters"]["dbt_commands"]) + + +def test_bundled_model_gates_on_tests_of_all_tested_upstreams(dbt_factory_bundled): + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "products"), + _model("pkg", "orders", depends_on=["model.pkg.customers", "model.pkg.products"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test("pkg", "unique_products_id", ["model.pkg.products"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "customers_test" in by_key + assert "products_test" in by_key + assert _commands(by_key["customers_test"]) == [ + "dbt test --select fqn:pkg.unique_customers_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["customers_test"]["depends_on"] == [{"task_key": "customers_model"}] + + # orders depends on both upstreams' bundled test tasks (rewired from their run tasks) + assert {dep["task_key"] for dep in by_key["orders_model"]["depends_on"]} == { + "customers_test", + "products_test", + } + + +def test_tests_on_seed_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _seed("pkg", "countries"), + _model("pkg", "enriched", depends_on=["seed.pkg.countries"]), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "countries_test" in by_key + assert _commands(by_key["countries_test"]) == [ + "dbt test --select fqn:pkg.unique_countries_code,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["countries_test"]["depends_on"] == [{"task_key": "countries_seed"}] + assert by_key["enriched_model"]["depends_on"] == [{"task_key": "countries_test"}] + + +def test_tests_on_snapshot_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _snapshot("pkg", "orders_snap"), + _model("pkg", "orders_history", depends_on=["snapshot.pkg.orders_snap"]), + _test("pkg", "not_null_orders_snap_id", ["snapshot.pkg.orders_snap"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "orders_snap_test" in by_key + assert _commands(by_key["orders_snap_test"]) == [ + "dbt test --select fqn:pkg.not_null_orders_snap_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["orders_snap_test"]["depends_on"] == [{"task_key": "orders_snap_snapshot"}] + assert by_key["orders_history_model"]["depends_on"] == [{"task_key": "orders_snap_test"}] + + +def test_tests_on_source_produce_standalone_task(dbt_factory_bundled): + nodes = dict( + [ + _test("pkg", "unique_raw_customers_id", ["source.pkg.raw.customers"]), + ] + ) + sources = dict([_source("pkg", "raw", "customers")]) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes, "sources": sources}) + by_key = {t["task_key"]: t for t in tasks} + + assert "raw_customers_test" in by_key + assert _commands(by_key["raw_customers_test"]) == [ + "dbt test --select fqn:pkg.unique_raw_customers_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["raw_customers_test"]["depends_on"] == [] + + +def test_flat_mode_emits_one_task_per_test_node_and_gates_downstream(dbt_factory): + # Per-test mode mirrors `dbt build`: downstream models wait on upstream tests, so a + # failing `severity: error` test skips downstream via Databricks task failure. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test("pkg", "not_null_customers_id", ["model.pkg.customers"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "unique_customers_id_test" in by_key + assert "not_null_customers_id_test" in by_key + assert "customers_test" not in by_key # no bundling in per-test mode + + assert _commands(by_key["unique_customers_id_test"]) == [ + "dbt test --select fqn:pkg.unique_customers_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["unique_customers_id_test"]["depends_on"] == [{"task_key": "customers_model"}] + # orders depends on customers AND every test attached to customers + assert {dep["task_key"] for dep in by_key["orders_model"]["depends_on"]} == { + "customers_model", + "unique_customers_id_test", + "not_null_customers_id_test", + } + + +def test_flat_mode_cross_model_test_does_not_create_cycle(dbt_factory): + # Relationship test references BOTH `orders` and `customers`. Without care, extending + # `orders`'s deps with "tests of upstream (customers)" would pull in the relationship test, + # which itself depends on `orders` — a direct cycle. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _model("pkg", "payments", depends_on=["model.pkg.orders"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test( + "pkg", + "relationships_orders_customer_id__ref_customers", + ["model.pkg.orders", "model.pkg.customers"], + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # orders depends on customers + unique_customers_id, but NOT on the relationship test + # (that test references orders itself — including it would cycle) + assert {dep["task_key"] for dep in by_key["orders_model"]["depends_on"]} == { + "customers_model", + "unique_customers_id_test", + } + + # payments (downstream of orders) picks up the relationship test — safe, payments + # transitively depends on both orders and customers (the test's refs) + payments_deps = {dep["task_key"] for dep in by_key["payments_model"]["depends_on"]} + assert "orders_model" in payments_deps + assert "relationships_orders_customer_id__ref_customers_test" in payments_deps + + +def test_flat_mode_transitive_cross_model_test_does_not_create_cycle(dbt_factory): + # Transitive cycle case: test T refs {A, C} where C is downstream of B which is downstream + # of A. Extending B's deps with "tests of upstream (A)" must NOT add T, because T depends + # on C and C depends on B → B → T → C → B cycle. Only nodes downstream of both A and C + # (i.e. downstream of C) should get T. + nodes = dict( + [ + _model("pkg", "a"), + _model("pkg", "b", depends_on=["model.pkg.a"]), + _model("pkg", "c", depends_on=["model.pkg.b"]), + _model("pkg", "d", depends_on=["model.pkg.c"]), + _test("pkg", "relationship_a_c", ["model.pkg.a", "model.pkg.c"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # B's ancestors = {A}. Test T refs = {A, C}. C ∉ ancestors(B) → skip T. + assert by_key["b_model"]["depends_on"] == [{"task_key": "a_model"}] + # C's ancestors = {A, B}. C IS in T.refs → skip T (direct self-reference). + assert by_key["c_model"]["depends_on"] == [{"task_key": "b_model"}] + # D's ancestors = {A, B, C}. T.refs = {A, C} ⊆ ancestors(D) → add T. + d_deps = {dep["task_key"] for dep in by_key["d_model"]["depends_on"]} + assert d_deps == {"c_model", "relationship_a_c_test"} + + +def test_flat_mode_tests_gate_downstream_regardless_of_severity(dbt_factory): + # Gating follows dbt's dependency graph, not a test's severity: a warn test can still fail the + # run under `--warn-error`, so both warn- and error-severity tests gate their downstream models. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"], severity="warn"), + _test("pkg", "not_null_customers_id", ["model.pkg.customers"], severity="error"), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "unique_customers_id_test" in by_key + assert "not_null_customers_id_test" in by_key + + # orders gates on customers and both attached tests, warn and error alike + assert {dep["task_key"] for dep in by_key["orders_model"]["depends_on"]} == { + "customers_model", + "unique_customers_id_test", + "not_null_customers_id_test", + } + + +def test_flat_mode_test_on_seed_gates_on_seed(dbt_factory): + nodes = dict( + [ + _seed("pkg", "countries"), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert by_key["unique_countries_code_test"]["depends_on"] == [{"task_key": "countries_seed"}] + + +def test_bundled_task_factory_assembles_commands(dbt_factory_bundled): + test_factory = dbt_factory_bundled.task_factories["test"] + task = test_factory.create_bundled_task( + task_key="customers_test", + selects_by_indirect_selection={"cautious": ["fqn:pkg.unique_customers_id,package:pkg,resource_type:test"]}, + deps_command_name="customers", + depends_on=["customers_model"], + ) + assert task.task_key == "customers_test" + assert task.commands == [ + "dbt test --select fqn:pkg.unique_customers_id,package:pkg,resource_type:test --target dev --indirect-selection cautious" + ] + assert task.depends_on == ["customers_model"] + + +def test_cross_model_test_in_bundled_mode_is_emitted_as_standalone_task(dbt_factory_bundled): + # The relationship test spans two models, so it must NOT be collapsed into either model's + # bundled test task (dbt would hit a TABLE_OR_VIEW_NOT_FOUND on the un-built endpoint). + # It should emit its own task with deps on both referenced models. + nodes = dict( + [ + _model("pkg", "team_cities"), + _model("pkg", "game_details", depends_on=["model.pkg.team_cities"]), + _test("pkg", "not_null_team_cities_name", ["model.pkg.team_cities"]), + _test( + "pkg", + "relationships_game_details_winner__team_city__ref_team_cities_", + ["model.pkg.game_details", "model.pkg.team_cities"], + ), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # Single-model test → its own bundled task addressing the exact test node + assert "team_cities_test" in by_key + assert _commands(by_key["team_cities_test"]) == [ + "dbt test --select fqn:pkg.not_null_team_cities_name,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + + # Cross-model test → its own task, gated on BOTH referenced models + cross_test_key = "relationships_game_details_winner__team_city__ref_team_cities__test" + assert cross_test_key in by_key + assert _commands(by_key[cross_test_key]) == [ + "dbt test --select fqn:pkg.relationships_game_details_winner__team_city__ref_team_cities_,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert {dep["task_key"] for dep in by_key[cross_test_key]["depends_on"]} == { + "team_cities_model", + "game_details_model", + } + + # `game_details` has no single-model tests, so no bundled `game_details_test` exists + assert "game_details_test" not in by_key + + +def test_single_package_bundled_test_uses_qualified_select(dbt_factory_bundled): + nodes = dict( + [ + _model("pkg_a", "customers"), + _model("pkg_a", "orders", depends_on=["model.pkg_a.customers"]), + _test("pkg_a", "unique_customers_id", ["model.pkg_a.customers"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "customers_test" in by_key + assert _commands(by_key["customers_test"]) == [ + "dbt test --select fqn:pkg_a.unique_customers_id,package:pkg_a,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["orders_model"]["depends_on"] == [{"task_key": "customers_test"}] + + +def test_flat_mode_two_tests_sharing_a_name_and_fqn_are_refused(dbt_factory): + # dbt lets the same custom test name land twice with the same fqn, telling the nodes apart only + # via the unique_id hash — which dbt exposes as no selector method. The factory cannot prove a + # selector addresses one without also running the other before its own dependencies are ready, + # so it refuses rather than emit a task that runs the wrong node. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "dup_check", ["model.pkg.customers"], test_hash="6ea6b2ac82"), + _test("pkg", "dup_check", ["model.pkg.customers"], test_hash="a9ab3a6e12"), + ] + ) + + with pytest.raises(ValueError, match="also runs"): + dbt_factory.create_tasks({"nodes": nodes}) + + +def test_flat_mode_cross_package_models_get_package_prefixed_keys(dbt_factory): + # Two packages may ship a model with the same name (materialized to different schemas). + nodes = dict( + [ + _model("shop", "stg_orders"), + _model("subpkg", "stg_orders"), + _model("shop", "mart", depends_on=["model.shop.stg_orders", "model.subpkg.stg_orders"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "shop_stg_orders_model" in by_key + assert "subpkg_stg_orders_model" in by_key + assert {dep["task_key"] for dep in by_key["mart_model"]["depends_on"]} == { + "shop_stg_orders_model", + "subpkg_stg_orders_model", + } + + +def test_bundled_mode_singular_test_named_like_tested_model_keeps_keys_unique(dbt_factory_bundled): + # The bundled test task for model `orders` claims `orders_test`, and so does a singular + # test file named `orders`. Both stay unique (and deploy cannot fail on a duplicate key). + nodes = dict( + [ + _model("pkg", "orders"), + _test("pkg", "unique_orders_id", ["model.pkg.orders"], test_hash="9a1b2c3d4e"), + _test("pkg", "orders", []), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + keys = [t["task_key"] for t in tasks] + by_key = {t["task_key"]: t for t in tasks} + + assert len(keys) == len(set(keys)) + assert _commands(by_key["pkg_orders_test"]) == [ + "dbt test --select fqn:pkg.unique_orders_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert _commands(by_key["pkg_orders_test_2"]) == [ + "dbt test --select fqn:pkg.orders,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + + +def test_generated_tasks_match_expected(dbt_factory): + """Safety check: the full set of tasks generated for the sample manifest matches the saved + copy in test_data/expected_tasks.json. If you change the generated output on purpose, refresh + that file with `make test-update-expected-tasks`. + """ + manifest = read_dbt_manifest(BASE_PATH + "/test_data/manifest.json") + tasks = dbt_factory.create_tasks(manifest) + + expected = json.loads(Path(BASE_PATH + "/test_data/expected_tasks.json").read_text()) + assert tasks == expected diff --git a/contrib/dbt_factory/tests/test_dbt_task.py b/contrib/dbt_factory/tests/test_dbt_task.py new file mode 100644 index 00000000..cb7a24ca --- /dev/null +++ b/contrib/dbt_factory/tests/test_dbt_task.py @@ -0,0 +1,45 @@ +import json + +from databricks_dbt_factory.dbt_task import DbtTask, DbtTaskOptions + + +def test_renders_serverless_notebook_task(): + options = DbtTaskOptions( + environment_key="Default", + notebook_path="./notebooks/dbt_runner.py", + project_directory="/project", + profiles_directory="/profiles", + ) + task = DbtTask( + task_key="my_model", + commands=["dbt run --select my_model --target dev"], + options=options, + depends_on=["upstream_model"], + ) + + result = task.to_dict() + + assert result["task_key"] == "my_model" + assert result["environment_key"] == "Default" # serverless + assert "job_cluster_key" not in result + assert result["depends_on"] == [{"task_key": "upstream_model"}] + + notebook_task = result["notebook_task"] + assert notebook_task["notebook_path"] == "./notebooks/dbt_runner.py" + assert notebook_task["base_parameters"]["project_directory"] == "/project" + assert notebook_task["base_parameters"]["profiles_directory"] == "/profiles" + assert json.loads(notebook_task["base_parameters"]["dbt_commands"]) == ["dbt run --select my_model --target dev"] + + +def test_defaults_and_optional_directories_are_omitted(): + options = DbtTaskOptions(notebook_path="./runner.py") + task = DbtTask(task_key="m", commands=["dbt run --select m"], options=options) + + result = task.to_dict() + + assert result["environment_key"] == "Default" # default serverless environment + assert result["depends_on"] == [] + base_parameters = result["notebook_task"]["base_parameters"] + assert "project_directory" not in base_parameters + assert "profiles_directory" not in base_parameters + assert json.loads(base_parameters["dbt_commands"]) == ["dbt run --select m"] diff --git a/contrib/dbt_factory/tests/test_load_resources.py b/contrib/dbt_factory/tests/test_load_resources.py new file mode 100644 index 00000000..aa102768 --- /dev/null +++ b/contrib/dbt_factory/tests/test_load_resources.py @@ -0,0 +1,121 @@ +""" +Offline test for the PyDABs integration in resources/__init__.py. + +Exercises the whole glue path against the committed dbt manifest (no Databricks workspace +required): reading the manifest, wiring the factory, generating notebook tasks, and registering +the job on a Resources object (which runs Job.from_dict, validating the task dict schema). +""" + +import json +from importlib.metadata import version +from pathlib import Path + +import pytest +import yaml +from databricks.bundles.core import Bundle + +import resources +from databricks_dbt_factory.utils import generate_task_key + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +# The manifest is generated by `make manifest` (dbt parse). In a freshly initialized project it +# does not exist yet, so skip these tests rather than erroring until the user has run it. +pytestmark = pytest.mark.skipif( + not (PROJECT_ROOT / resources.MANIFEST_PATH).exists(), + reason="dbt manifest not found; run `make manifest` (dbt parse) first", +) + + +@pytest.fixture(autouse=True) +def _chdir_to_project_root(monkeypatch): + # resources.MANIFEST_PATH is relative to the bundle root, as it is at deploy time. + monkeypatch.chdir(PROJECT_ROOT) + + +def test_build_tasks_covers_every_buildable_node(): + # Every model/seed/snapshot node gets its own run task regardless of the BUNDLE_TESTS mode + # (tests may be collapsed into per-resource `_test` tasks when bundling is on). + tasks = resources._build_tasks("dev") + task_keys = {task["task_key"] for task in tasks} + + manifest = json.loads((PROJECT_ROOT / resources.MANIFEST_PATH).read_text()) + buildable = [ + uid + for uid, node in manifest.get("nodes", {}).items() + if node.get("resource_type") in {"model", "seed", "snapshot"} + ] + expected_keys = {generate_task_key(uid) for uid in buildable} + + assert buildable, "committed manifest has no model/seed/snapshot nodes" + assert expected_keys <= task_keys + + +def test_generated_tasks_are_serverless_notebook_tasks(): + tasks = resources._build_tasks("dev") + + for task in tasks: + assert "notebook_task" in task, task + assert task["environment_key"] == resources.ENVIRONMENT_KEY + assert "job_cluster_key" not in task # serverless only + notebook_task = task["notebook_task"] + assert notebook_task["notebook_path"] == resources.RUNNER_NOTEBOOK_PATH + # The dbt target is injected into every command. + assert "--target dev" in notebook_task["base_parameters"]["dbt_commands"] + + +def test_environment_pins_installed_dbt_databricks(): + # Single source of truth: the serverless env pins dbt-databricks to the exact + # version installed in the bundle venv (the one used to generate the manifest). + spec = resources._serverless_environment_spec() + assert spec["dependencies"] == [f"dbt-databricks=={version('dbt-databricks')}"] + + # The generated job points its serverless environment at the synced env file. + job = resources.build_job("dev") + assert job.environments[0].spec.base_environment == resources.SERVERLESS_ENV_PATH + + +def test_load_resources_registers_the_job(): + result = resources.load_resources(Bundle(target="dev")) + try: + assert resources.JOB_NAME in result.jobs + job = result.jobs[resources.JOB_NAME] + assert job.environments[0].environment_key == resources.ENVIRONMENT_KEY + + # load_resources writes the base-environment file to the bundle root for + # the bundle to sync; it pins dbt-databricks to the installed version. + env_file = PROJECT_ROOT / resources.SERVERLESS_ENV_FILE + assert env_file.exists() + content = yaml.safe_load(env_file.read_text()) + pin = f"dbt-databricks=={version('dbt-databricks')}" + assert content["dependencies"] == [pin] + finally: + (PROJECT_ROOT / resources.SERVERLESS_ENV_FILE).unlink(missing_ok=True) + + +def test_dependency_pin_rejects_non_pypi_versions(monkeypatch): + # A local or dev build cannot be pip-installed from PyPI when Databricks builds the + # serverless environment, so the deploy must fail early with a clear message. + for installed in ("1.9.0+custom", "1.13.0.dev0", "not-a-version"): + monkeypatch.setattr(resources, "version", lambda name, v=installed: v) + with pytest.raises(RuntimeError, match="PyPI"): + resources._dbt_databricks_dependency() + + +def test_environment_file_write_is_idempotent(): + env_file = PROJECT_ROOT / resources.SERVERLESS_ENV_FILE + try: + resources._write_serverless_environment_file() + env_file.chmod(0o444) + # Content already matches: no write is attempted, so a read-only file is fine. + resources._write_serverless_environment_file() + env_file.chmod(0o644) + + env_file.write_text("stale: true\n") + resources._write_serverless_environment_file() + pin = f"dbt-databricks=={version('dbt-databricks')}" + assert yaml.safe_load(env_file.read_text())["dependencies"] == [pin] + finally: + if env_file.exists(): + env_file.chmod(0o644) + env_file.unlink() diff --git a/contrib/dbt_factory/tests/test_utils.py b/contrib/dbt_factory/tests/test_utils.py new file mode 100644 index 00000000..0a698139 --- /dev/null +++ b/contrib/dbt_factory/tests/test_utils.py @@ -0,0 +1,93 @@ +from databricks_dbt_factory.utils import ( + MAX_TASK_KEY_LENGTH, + build_task_key_maps, + bundled_test_key, + generate_task_key, +) + + +def test_resource_keys_use_name_and_type_suffix(): + assert generate_task_key("model.shop.orders") == "orders_model" + assert generate_task_key("seed.shop.countries") == "countries_seed" + assert generate_task_key("snapshot.shop.orders_snap") == "orders_snap_snapshot" + + +def test_versioned_model_keeps_version(): + assert generate_task_key("model.shop.dim_customers.v2") == "dim_customers_v2_model" + + +def test_test_key_drops_package_and_hash(): + assert generate_task_key("test.shop.unique_orders_id.9a1b2c3d4e") == "unique_orders_id_test" + # singular test (no trailing hash segment) + assert generate_task_key("test.shop.assert_positive_amount") == "assert_positive_amount_test" + + +def test_source_key(): + assert generate_task_key("source.shop.raw.customers") == "raw_customers_test" + + +def test_bundled_test_key(): + assert bundled_test_key("model.shop.orders") == "orders_test" + assert bundled_test_key("seed.shop.countries") == "countries_test" + assert bundled_test_key("source.shop.raw.customers") == "raw_customers_test" + + +def test_long_test_key_is_truncated_and_hash_disambiguated(): + long_name = "accepted_values_" + "x" * 200 + key = generate_task_key(f"test.shop.{long_name}.9a1b2c3d4e") + assert len(key) <= MAX_TASK_KEY_LENGTH + assert key.endswith("_9a1b2c3d4e_test") # hash re-added so truncated names stay unique + + +def test_key_maps_keep_plain_keys_when_unique(): + task_keys, bundled = build_task_key_maps(["model.shop.orders", "test.shop.unique_orders_id.9a1b2c3d4e"]) + assert task_keys == { + "model.shop.orders": "orders_model", + "test.shop.unique_orders_id.9a1b2c3d4e": "unique_orders_id_test", + } + assert bundled == {} + + +def test_key_maps_disambiguate_same_named_tests_with_dbt_hash(): + # The same custom test name on two models: dbt keeps them apart only via the unique_id hash. + task_keys, _ = build_task_key_maps(["test.shop.dup_check.6ea6b2ac82", "test.shop.dup_check.a9ab3a6e12"]) + assert task_keys["test.shop.dup_check.6ea6b2ac82"] == "dup_check_6ea6b2ac82_test" + assert task_keys["test.shop.dup_check.a9ab3a6e12"] == "dup_check_a9ab3a6e12_test" + + +def test_key_maps_disambiguate_singular_vs_custom_named_test(): + # A singular test file and a custom-named generic test can share a name; the singular test + # has no hash, so it gets the package folded in instead. + task_keys, _ = build_task_key_maps(["test.shop.raw_customers", "test.shop.raw_customers.e58cc24de2"]) + assert task_keys["test.shop.raw_customers"] == "shop_raw_customers_test" + assert task_keys["test.shop.raw_customers.e58cc24de2"] == "raw_customers_e58cc24de2_test" + + +def test_key_maps_disambiguate_cross_package_models_with_package_name(): + task_keys, _ = build_task_key_maps(["model.shop.a", "model.subpkg.a", "model.shop.b"]) + assert task_keys["model.shop.a"] == "shop_a_model" + assert task_keys["model.subpkg.a"] == "subpkg_a_model" + assert task_keys["model.shop.b"] == "b_model" + + +def test_key_maps_disambiguate_bundled_test_key_against_task_keys(): + # Bundled mode: model `orders` claims the bundled key `orders_test`, and so does a singular + # test named `orders`. Every returned key must still be unique. + task_keys, bundled = build_task_key_maps( + ["model.shop.orders", "test.shop.orders"], bundled_test_ids=["model.shop.orders"] + ) + all_keys = list(task_keys.values()) + list(bundled.values()) + assert len(all_keys) == len(set(all_keys)) + assert task_keys["model.shop.orders"] == "orders_model" + assert task_keys["test.shop.orders"] == "shop_orders_test_2" + assert bundled["model.shop.orders"] == "shop_orders_test" + + +def test_key_maps_disambiguated_long_test_keys_keep_hash_within_limit(): + long_name = "accepted_values_" + "y" * 200 + ids = [f"test.shop.{long_name}.6ea6b2ac82", f"test.shop.{long_name}.a9ab3a6e12"] + task_keys, _ = build_task_key_maps(ids) + assert len(set(task_keys.values())) == 2 + for uid, key in task_keys.items(): + assert len(key) <= MAX_TASK_KEY_LENGTH + assert key.endswith(f"_{uid.split('.')[3]}_test") diff --git a/contrib/dbt_factory/tests/update_expected_tasks.py b/contrib/dbt_factory/tests/update_expected_tasks.py new file mode 100644 index 00000000..95ea91d3 --- /dev/null +++ b/contrib/dbt_factory/tests/update_expected_tasks.py @@ -0,0 +1,30 @@ +"""Refresh tests/test_data/expected_tasks.json — the saved copy of the job tasks the factory +generates for the sample manifest. The test `test_generated_tasks_match_expected` compares the +current output against this file, so run this script (via `make test-update-expected-tasks`) whenever +you intentionally change the generated output. + +It builds the factory the exact same way the tests do (conftest.create_dbt_factory), so the saved +copy can't drift from what the tests expect. +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from conftest import create_dbt_factory # noqa: E402 +from databricks_dbt_factory.utils import read_dbt_manifest # noqa: E402 + +TEST_DATA = Path(__file__).resolve().parent / "test_data" + + +def main() -> None: + manifest = read_dbt_manifest(str(TEST_DATA / "manifest.json")) + tasks = create_dbt_factory().create_tasks(manifest) + (TEST_DATA / "expected_tasks.json").write_text(json.dumps(tasks, indent=2) + "\n") + print(f"Wrote {len(tasks)} tasks to {TEST_DATA / 'expected_tasks.json'}") + + +if __name__ == "__main__": + main() diff --git a/contrib/templates/dbt-factory/.ruff.toml b/contrib/templates/dbt-factory/.ruff.toml new file mode 100644 index 00000000..af8c4f89 --- /dev/null +++ b/contrib/templates/dbt-factory/.ruff.toml @@ -0,0 +1,5 @@ +# Ruff config for the template's Python sources. It lives here — a sibling of the template +# schema, OUTSIDE template/ — so `databricks bundle init` never renders it into a generated +# project, yet the repo-root `ruff format --check` still discovers it (as the closest config) +# for the sources under template/. Matches the line-length the generated pyproject.toml uses. +line-length = 120 diff --git a/contrib/templates/dbt-factory/README.md b/contrib/templates/dbt-factory/README.md new file mode 100644 index 00000000..16327780 --- /dev/null +++ b/contrib/templates/dbt-factory/README.md @@ -0,0 +1,95 @@ +# dbt-factory template + +A [Declarative Automation Bundles](https://docs.databricks.com/dev-tools/bundles/index.html) template +that generates a [dbt](https://docs.getdbt.com/) project whose Databricks Workflow is built +**from the dbt manifest at deploy time** — one Databricks task per dbt object (model, seed, +snapshot, test), running on serverless compute. + +It wires together two pieces: + +* **[databricks-dbt-factory](https://github.com/mwojtyczka/databricks-dbt-factory)** — expands a + dbt `manifest.json` into Databricks job tasks with their dependencies. Its source is included + in every generated project under `src/databricks_dbt_factory/`. +* **[PyDABs](https://docs.databricks.com/dev-tools/bundles/python)** — the `python.resources` + hook. At `databricks bundle deploy` time the Databricks CLI calls `load_resources`, which runs + the factory against the manifest and returns the generated job. No per-model job YAML is checked + in. + +Instead of running the whole dbt project as one opaque task, you get: + +* **Faster execution** — independent models run in parallel; the notebook task type keeps dbt's + dependencies pre-cached in the serverless environment, avoiding a per-task cold start. +* **Visibility & simplified troubleshooting** — pinpoint failures at the model level in the UI. +* **Enhanced logging & notifications** — per-task logs and precise, model-level error alerts. +* **Improved retriability** — retry only the failed model tasks, not the whole project. +* **Seamless testing** — dbt data tests run as their own tasks right after each model finishes. + +For a pre-initialized, ready-to-read version of what this template produces, see the +[`contrib/dbt_factory`](../../dbt_factory) example. + +## How it works + +`databricks bundle init` scaffolds a self-contained project; each `databricks bundle deploy` then +regenerates the Workflow from your current dbt manifest, so adding or removing a model just works +on the next deploy — no per-model YAML to maintain. + +```mermaid +flowchart TD + subgraph setup["One-time setup"] + T["dbt-factory bundle template"] -->|databricks bundle init| B["Scaffolded project:
dbt project + PyDABs hook + factory code"] + X["Existing dbt project
(optional)"] -.->|move models/seeds/... into src/| B + end + subgraph deploy["Every deploy"] + C["make manifest
(dbt parse)"] --> D["target/manifest.json"] + D --> E["databricks bundle deploy"] + E --> F["PyDABs load_resources reads the
manifest and generates the job"] + end + subgraph runtime["At run time — serverless"] + G["Databricks Workflow:
one task per model / seed / snapshot / test"] --> H["Each task triggers dbt
via the runner notebook"] + H --> I[("SQL warehouse")] + end + B --> C + F --> G + + classDef optional stroke:#999,stroke-dasharray:5 4,color:#888; + class X optional; +``` + +## Usage + +``` +$ databricks bundle init https://github.com/databricks/bundle-examples --template-dir contrib/templates/dbt-factory +``` + +Answer the prompts (project name, catalog, dev schema, warehouse HTTP path, and a few +factory options). Then: + +``` +$ cd +$ make setup # install dependencies into .venv +$ make manifest # generate the dbt manifest (dbt parse) — required before the first deploy +$ databricks bundle deploy --target dev +$ databricks bundle run _job +``` + +## Prompts + +| Prompt | Purpose | +|---|---| +| `project_name` | Bundle / dbt project name; also names the generated job `_job`. | +| `default_catalog` | Unity Catalog catalog dbt writes to. | +| `dev_schema` | Schema for the `dev` target (`prod` uses `default`). | +| `http_path` | HTTP path of the SQL warehouse dbt connects to. | +| `bundle_tests` | Bundle single-model tests per resource into one task (performance boost). | +| `environment_key` | Key of the serverless environment used by the generated job. | +| `extra_dbt_command_options` | Extra options appended to every generated dbt command. | + +## Already have a dbt project? + +This template scaffolds a new project. To reuse an **existing** dbt project, generate a project +from this template and move your dbt files into it — see the +["Migrating an existing dbt project"](../../dbt_factory/README.md#migrating-an-existing-dbt-project) +guide in the example. You won't need to bring any dependencies or edit paths. + +See https://github.com/databricks/bundle-examples/blob/main/contrib/README.md for more about +community contributions. diff --git a/contrib/templates/dbt-factory/databricks_template_schema.json b/contrib/templates/dbt-factory/databricks_template_schema.json new file mode 100644 index 00000000..53d615fc --- /dev/null +++ b/contrib/templates/dbt-factory/databricks_template_schema.json @@ -0,0 +1,57 @@ +{ + "welcome_message": "\nWelcome to the dbt-factory template for Declarative Automation Bundles!\nThis generates a dbt project whose Databricks Workflow is built from the dbt manifest at deploy time — one task per dbt object (model, seed, snapshot, test), on serverless.", + "properties": { + "project_name": { + "type": "string", + "default": "my_dbt_project", + "description": "\nPlease provide the following details to tailor the template to your preferences.\n\nUnique name for this project\nproject_name", + "order": 1, + "pattern": "^[A-Za-z0-9_]+$", + "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." + }, + "default_catalog": { + "type": "string", + "default": "{{default_catalog}}", + "pattern": "^\\w*$", + "pattern_match_failure_message": "Invalid catalog name.", + "description": "\nInitial catalog dbt writes to{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}.\ndefault_catalog", + "order": 2 + }, + "dev_schema": { + "type": "string", + "default": "default", + "pattern": "^\\w+$", + "pattern_match_failure_message": "Invalid schema name.", + "description": "\nSchema dbt writes to for the 'dev' target (the 'prod' target uses 'default').\ndev_schema", + "order": 3 + }, + "http_path": { + "type": "string", + "default": "/sql/1.0/warehouses/abcdef1234567890", + "description": "\nHTTP path of the SQL warehouse dbt connects to.\nhttp_path", + "order": 4 + }, + "bundle_tests": { + "type": "string", + "default": "no", + "enum": ["no", "yes"], + "description": "\nBundle single-model tests per resource into one dbt test task? This is a performance boost for projects with many tests (fewer Databricks task startups), at the cost of per-test visibility in the UI.\nbundle_tests", + "order": 5 + }, + "environment_key": { + "type": "string", + "default": "Default", + "pattern": "^\\w+$", + "pattern_match_failure_message": "Invalid environment key.", + "description": "\nKey of the serverless environment used by the generated job.\nenvironment_key", + "order": 6 + }, + "extra_dbt_command_options": { + "type": "string", + "default": "", + "description": "\nAdditional options appended to every generated dbt command (e.g. \"--full-refresh\"). Selection and parse-context options (--select, --vars, --target, ...) are rejected. Leave blank if unsure.\nextra_dbt_command_options", + "order": 7 + } + }, + "success_message": "\n\nYour new project has been created in the '{{.project_name}}' directory!\n\nNext steps:\n\n $ cd {{.project_name}}\n $ make setup # install dependencies into .venv\n $ make manifest # generate the dbt manifest (dbt parse) — required before the first deploy\n $ databricks bundle deploy --target dev\n\nRefer to the README.md file for full instructions." +} diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/.gitignore b/contrib/templates/dbt-factory/template/{{.project_name}}/.gitignore new file mode 100644 index 00000000..5a79248b --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/.gitignore @@ -0,0 +1,19 @@ +.venv/ +__pycache__/ +*.pyc +.databricks/ +logs/ +# Ignore installed dbt packages but keep the directory (via .gitkeep) so the +# `dbt_packages/**` sync.include in databricks.yml always matches at least one file. +dbt_packages/* +!dbt_packages/.gitkeep +dbt_profiles/.user.yml +uv.lock + +# dbt build artifacts, but keep the committed manifest that resources/__init__.py reads at +# deploy time. Regenerate it with `make manifest`. +target/* +!target/manifest.json + +# Serverless base-environment file, regenerated on every `bundle deploy` by resources/__init__.py. +dbt_serverless_env.yaml diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/Makefile.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/Makefile.tmpl new file mode 100644 index 00000000..b4a124e0 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/Makefile.tmpl @@ -0,0 +1,35 @@ +.PHONY: setup deps manifest validate deploy run test test-update-expected-tasks + +# Install dependencies into the .venv used by the bundle (databricks.yml -> python.venv_path). +setup: + uv sync --dev + +# Install dbt package dependencies declared in packages.yml / dependencies.yml (if any). +# `dbt deps` empties dbt_packages/, so restore the .gitkeep afterwards; it keeps the directory +# present so the `dbt_packages/**` sync.include in databricks.yml always matches at least one file. +deps: + uv run dbt deps + touch dbt_packages/.gitkeep + +# Regenerate the dbt manifest that resources/__init__.py reads at deploy time. +# `dbt parse` does not connect to a warehouse; it only reads the project files. +manifest: deps + uv run dbt parse --profiles-dir dbt_profiles + +validate: + databricks bundle validate + +# Regenerate the manifest and deploy the generated job to the dev target. +deploy: manifest + databricks bundle deploy --target dev + +run: + databricks bundle run {{.project_name}}_job + +test: + uv run pytest tests + +# Refresh the saved "expected tasks" file the tests compare against, after you intentionally +# change the generated output. +test-update-expected-tasks: + uv run python tests/update_expected_tasks.py diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/NOTICE b/contrib/templates/dbt-factory/template/{{.project_name}}/NOTICE new file mode 100644 index 00000000..702735ef --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/NOTICE @@ -0,0 +1,34 @@ +This project includes code adapted from the "databricks-dbt-factory" library. + + Source: https://github.com/mwojtyczka/databricks-dbt-factory + Full history: https://github.com/mwojtyczka/databricks-dbt-factory/commits/main + Adapted from: commit 29866edb82de28c33fe888be67d451068d4d7f94 (v0.3.5) + Location: src/databricks_dbt_factory/ + +The code under src/databricks_dbt_factory/ originates from that repository (trimmed to the +serverless + notebook-task subset used here, then reformatted to this repository's style) and is +provided under the MIT +license reproduced below, which this NOTICE preserves for attribution. All other files are +part of the bundle-examples repository and are covered by that repository's license. + +-------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2024-present mwojtyczka + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/README.md.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/README.md.tmpl new file mode 100644 index 00000000..bfbed354 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/README.md.tmpl @@ -0,0 +1,103 @@ +# {{.project_name}} + +The '{{.project_name}}' project was generated using the **dbt-factory** template for +Declarative Automation Bundles. It runs a [dbt](https://docs.getdbt.com/) project on Databricks as a +Databricks Workflow with **one task per dbt object** (model, seed, snapshot, test), instead of +running the whole project as a single opaque task. + +It combines two pieces: + +* **[databricks-dbt-factory](https://github.com/mwojtyczka/databricks-dbt-factory)** — reads a + dbt `manifest.json` and expands it into Databricks job tasks with their dependencies wired up. + Vendored under `src/databricks_dbt_factory/` (see [`NOTICE`](NOTICE)). +* **[PyDABs](https://docs.databricks.com/dev-tools/bundles/python)** — at `bundle deploy` time + the Databricks CLI calls `load_resources` in [`resources/__init__.py`](resources/__init__.py), + which runs the factory against the manifest and returns the generated job. **No per-model job + YAML is checked in** — the task graph is generated on the fly from the manifest. + +This project uses **serverless compute** and the **notebook task type** for the fastest task +start times. + +## Setup + +1. Install the [Databricks CLI](https://docs.databricks.com/dev-tools/cli/databricks-cli.html) + and the [uv](https://docs.astral.sh/uv/) package manager. + +2. Authenticate to your Databricks workspace: + ``` + $ databricks configure + ``` + +3. Install dependencies into the `.venv` the bundle uses: + ``` + $ make setup # == uv sync --dev + ``` + +4. Check the SQL warehouse `http_path`, `catalog`, and `schema` in `dbt_profiles/profiles.yml`. + +## Generate the dbt manifest (required before the first deploy) + +`resources/__init__.py` reads the dbt manifest at deploy time. Generate it from your models: + +``` +$ make manifest # == uv run dbt deps && uv run dbt parse +``` + +`dbt parse` only reads your project files; it does not connect to a warehouse. Rerun it whenever +you change your models. It also writes `target/partial_parse.msgpack` — the bundle syncs it and +each task injects it to skip dbt's parse phase at runtime (a big speedup on large projects; it +works because the runtime dbt is pinned to your local version). The manifest location is +configurable via the `DBT_MANIFEST_PATH` env var or the `MANIFEST_PATH` constant in +`resources/__init__.py`. + +## Deploy and run + +``` +$ make deploy # regenerates the manifest + parse cache, then deploys +$ make run # == databricks bundle run {{.project_name}}_job +``` + +Or call the CLI directly (run `make manifest` first so the manifest + parse cache are current): + +``` +$ databricks bundle deploy --target dev +$ databricks bundle run {{.project_name}}_job +``` + +Open the run URL the CLI prints to watch the generated per-model task graph execute. + +## Configuring the generated job + +The knobs you chose at init are constants at the top of `resources/__init__.py` +(`BUNDLE_TESTS`, `ENVIRONMENT_KEY`, `EXTRA_DBT_COMMAND_OPTIONS`); edit them there to change +behavior. The dbt target, warehouse, catalog, and schema live in `dbt_profiles/profiles.yml` and +are selected per bundle target via `--target ${bundle.target}`. + +The runtime dbt-databricks version is not a knob: at deploy time `resources/__init__.py` pins the +serverless environment to the exact `dbt-databricks` installed in your `.venv` (the version that +generated the manifest), shipped as a `dbt_serverless_env.yaml` [base environment](https://docs.databricks.com/aws/en/compute/serverless/dependencies) +the bundle generates and syncs each deploy. `pyproject.toml` is the single source of truth: change +the version there, re-run `make setup`, and the next deploy uses it — so what runs in Databricks +matches what you tested with. + +## Tests + +``` +$ make test # == uv run pytest tests (run `make manifest` first) +``` + +Runs the vendored factory's test suite plus an offline test of the PyDABs integration against the +generated manifest. One test compares the generated tasks with a saved snapshot +(`tests/test_data/expected_tasks.json`); after an intentional change to the generated output, +refresh it with `make test-update-expected-tasks`. + +## Folding in an existing dbt project + +Already have a dbt project you'd rather use? Just move your dbt sources into this project's `src/` +folders (`src/models`, `src/seeds`, …) in place of the starter models, then run `make manifest`. +If you use dbt packages, copy your `packages.yml` to the project root too: `make manifest` +installs them (`dbt deps`) and the bundle syncs the resulting `dbt_packages/` to the workspace, +so the job never installs packages at runtime. +See the +[Migrating an existing dbt project](https://github.com/databricks/bundle-examples/blob/main/contrib/dbt_factory/README.md#migrating-an-existing-dbt-project) +guide for the full steps. diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/databricks.yml.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/databricks.yml.tmpl new file mode 100644 index 00000000..51a11a69 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/databricks.yml.tmpl @@ -0,0 +1,52 @@ +# This is a Declarative Automation Bundles definition for {{.project_name}}. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +# +# The job for this bundle is NOT defined in YAML. It is generated at deploy time from the dbt +# manifest by resources/__init__.py (the `python.resources` hook below), producing one +# Databricks task per dbt object (model / seed / snapshot / test). +bundle: + name: {{.project_name}} + +# PyDABs: the Databricks CLI calls `load_resources` during `bundle deploy` to build resources +# defined in Python. See resources/__init__.py. +python: + venv_path: .venv + resources: + - "resources:load_resources" + +# These files are generated (by resources/__init__.py / `make manifest`) and git-ignored; include +# them explicitly so `bundle deploy` still syncs them to the workspace: +# - dbt_serverless_env.yaml the base_environment the generated job points at. +# - target/partial_parse.msgpack dbt's parse cache; each task injects it to skip parsing (see +# README). Regenerated by `make manifest`; usable at runtime +# because the runtime dbt is pinned to your local version. +# - dbt_packages/** dbt packages installed by `make manifest` (dbt deps), so tasks +# never install packages at runtime. +sync: + include: + - dbt_serverless_env.yaml + - target/partial_parse.msgpack + - dbt_packages/** + +# Deployment targets. The dbt target is selected via `--target ${bundle.target}`, so these +# names match the dbt outputs in dbt_profiles/profiles.yml. +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # See https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: {{workspace_host}} + + prod: + mode: production + workspace: + host: {{workspace_host}} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + permissions: + - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} + level: CAN_MANAGE + run_as: + {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_packages/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_packages/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl new file mode 100644 index 00000000..5e98d38b --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl @@ -0,0 +1,36 @@ +# This file defines dbt profiles for deployed dbt jobs. +# The generated Databricks job selects the target via `--target ${bundle.target}`, +# so the dbt target names below (dev / prod) match the bundle targets in databricks.yml. +{{.project_name}}: + target: dev # default target + outputs: + + # Doing local development with the dbt CLI? + # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init' + # (See README.md) + + # The default target when deployed with the Databricks CLI + dev: + type: databricks + method: http + catalog: {{.default_catalog}} + schema: {{.dev_schema}} + + http_path: {{.http_path}} + + # The workspace host / token are provided by the runner notebook at runtime + # (see src/databricks_dbt_factory/notebook/run_dbt_command.py). + host: "{{`{{`}} env_var('DBT_HOST', '') {{`}}`}}" + token: "{{`{{`}} env_var('DBT_ACCESS_TOKEN', '') {{`}}`}}" + + # The production target when deployed with the Databricks CLI + prod: + type: databricks + method: http + catalog: {{.default_catalog}} + schema: default + + http_path: {{.http_path}} + + host: "{{`{{`}} env_var('DBT_HOST', '') {{`}}`}}" + token: "{{`{{`}} env_var('DBT_ACCESS_TOKEN', '') {{`}}`}}" diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_project.yml.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_project.yml.tmpl new file mode 100644 index 00000000..a14419cc --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_project.yml.tmpl @@ -0,0 +1,28 @@ +name: '{{.project_name}}' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: '{{.project_name}}' + +# These configurations specify where dbt should look for different types of files. +# Everything dbt-related lives under src/ so the project can also hold non-dbt resources +# (such as the vendored databricks_dbt_factory library under src/databricks_dbt_factory). +model-paths: ["src/models"] +analysis-paths: ["src/analyses"] +test-paths: ["src/tests"] +seed-paths: ["src/seeds"] +macro-paths: ["src/macros"] +snapshot-paths: ["src/snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +# In this example config, we tell dbt to build all models in the example/ +# directory as views by default. These settings can be overridden in the +# individual model files using the `{{`{{`}} config(...) {{`}}`}}` macro. +models: + {{.project_name}}: + example: + +materialized: view diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/profile_template.yml b/contrib/templates/dbt-factory/template/{{.project_name}}/profile_template.yml new file mode 100644 index 00000000..f26ec0b3 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/profile_template.yml @@ -0,0 +1,23 @@ +# This file defines prompts with defaults for dbt initialization. +# It is used when the `dbt init` command is invoked. +# +fixed: + type: databricks +prompts: + host: + default: company.databricks.com + token: + hint: 'personal access token to use, dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + hide_input: true + http_path: + hint: 'HTTP path of SQL warehouse to use' + default: /sql/1.0/warehouses/abcdef1234567890 + catalog: + hint: 'initial catalog' + default: catalog + schema: + hint: 'personal schema where dbt will build objects during development, example: user_name' + threads: + hint: 'threads to use during development, 1 or more' + type: 'int' + default: 4 diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/pyproject.toml.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/pyproject.toml.tmpl new file mode 100644 index 00000000..624ed636 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/pyproject.toml.tmpl @@ -0,0 +1,32 @@ +[project] +name = "{{.project_name}}" +version = "0.0.1" +requires-python = ">=3.10,<3.13" +# Used at `bundle deploy` time by resources/__init__.py (to write the serverless +# base-environment file) and by the vendored databricks_dbt_factory core. +dependencies = [ + "PyYAML~=6.0.1", +] + +[dependency-groups] +dev = [ + # Required by the PyDABs hook (resources/__init__.py) at `bundle deploy` time. + "databricks-bundles==1.8.0", + # Single source of truth for the dbt-databricks version: used locally to generate the + # manifest (`make manifest`), and resources/__init__.py pins the serverless runtime to this + # exact installed version at deploy time. Pin/lock it for fully reproducible deploys. + "dbt-databricks>=1.8.0,<2.0.0", + "pytest", + "ruff", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +# Only the vendored library is packaged; the dbt project (src/models, ...) is not a package. +[tool.hatch.build.targets.wheel] +packages = ["src/databricks_dbt_factory"] + +[tool.ruff] +line-length = 120 diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/resources/__init__.py.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/resources/__init__.py.tmpl new file mode 100644 index 00000000..419ecf2b --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/resources/__init__.py.tmpl @@ -0,0 +1,195 @@ +""" +PyDABs integration for databricks-dbt-factory. + +At ``databricks bundle deploy`` time the Databricks CLI calls :func:`load_resources`, which +reads the dbt manifest and expands it into a Databricks job with one task per dbt object +(model / seed / snapshot / test) using the vendored ``databricks_dbt_factory`` core. No +per-model YAML is generated or checked in — the job graph is built on the fly from the manifest. + +This module is the only integration glue: it configures the vendored ``databricks_dbt_factory`` +core (under ``src/databricks_dbt_factory``) for serverless notebook tasks and returns a +``Resources`` object for PyDABs. +""" + +import os +from importlib.metadata import PackageNotFoundError, version + +import yaml +from databricks.bundles.core import Bundle, Resources +from databricks.bundles.jobs import Job + +from databricks_dbt_factory.dbt_factory import DbtFactory +from databricks_dbt_factory.dbt_task import DbtTaskOptions, TaskType +from databricks_dbt_factory.task_factory import ( + DbtDependencyResolver, + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, +) +from databricks_dbt_factory.utils import read_dbt_manifest + +# --- Configuration ------------------------------------------------------------------------- + +# Path to the dbt manifest, read at deploy time. Regenerate with `make manifest` (dbt parse). +# Override the location without editing this file via the DBT_MANIFEST_PATH env var. +MANIFEST_PATH = os.environ.get("DBT_MANIFEST_PATH", "target/manifest.json") + +# Name of the generated Databricks job. +JOB_NAME = "{{.project_name}}_job" + +# Key of the serverless environment (defined on the job below). +ENVIRONMENT_KEY = "{{.environment_key}}" + +# Bundle single-model tests per resource into one `dbt test` task (fewer task startups, +# faster end-to-end runtime for projects with many tests). See the databricks-dbt-factory +# README "DBT Tests handling" section for the trade-offs. +BUNDLE_TESTS = {{if eq .bundle_tests "yes"}}True{{else}}False{{end}} + +# Extra options appended to every generated dbt command (e.g. "--full-refresh"). +# Selection options (`--select`, `--exclude`, ...) and parse-context options (`--vars`, `--target`, +# `--profiles-dir`, `--project-dir`) are rejected: the factory owns selection and the runtime parse +# context must match the supplied manifest. +# `printf "%q"` renders a safely-quoted string literal so values containing quotes stay valid Python. +EXTRA_DBT_COMMAND_OPTIONS = {{ printf "%q" .extra_dbt_command_options }} + +# Serverless base-environment file, generated at deploy time from the pinned +# dbt-databricks version and synced with the bundle. The generated job points its +# serverless environment at this file (see build_job) so Databricks pre-builds the +# environment once instead of installing dbt on every task. `${workspace.file_path}` +# is substituted by the CLI at deploy time to the bundle's synced files root. +SERVERLESS_ENV_FILE = "dbt_serverless_env.yaml" +SERVERLESS_ENV_PATH = "${workspace.file_path}/" + SERVERLESS_ENV_FILE + +# The runner notebook shipped with the core, referenced in place (path relative to the bundle +# root). `PROJECT_DIRECTORY` is the path from the notebook's own directory back up to the dbt +# project root (== bundle root), where the runner changes directory to before running dbt. It is +# derived from the notebook path so the two stay consistent if the layout ever changes. +RUNNER_NOTEBOOK_PATH = "src/databricks_dbt_factory/notebook/run_dbt_command.py" +PROJECT_DIRECTORY = os.path.relpath(".", os.path.dirname(RUNNER_NOTEBOOK_PATH)) +PROFILES_DIRECTORY = "dbt_profiles" + + +def _build_tasks(target: str) -> list[dict]: + """Reads the dbt manifest and returns the list of Databricks task dicts (one per dbt node).""" + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + environment_key=ENVIRONMENT_KEY, + notebook_path=RUNNER_NOTEBOOK_PATH, + project_directory=PROJECT_DIRECTORY, + profiles_directory=PROFILES_DIRECTORY, + task_type=TaskType.NOTEBOOK, + ) + dbt_options = f"--target {target} {EXTRA_DBT_COMMAND_OPTIONS}".strip() + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + factory = DbtFactory(task_factories, bundle_tests=BUNDLE_TESTS) + manifest = read_dbt_manifest(MANIFEST_PATH) + return factory.create_tasks(manifest) + + +def _dbt_databricks_dependency() -> str: + """Pin the runtime to the exact dbt-databricks installed in this bundle's venv — the same + version used to generate the manifest (`make manifest`) and to develop locally. pyproject.toml + is therefore the single source of truth for the dbt version, and the version that runs in + Databricks is guaranteed to match the one you tested with (no separate range that can drift). + Local or dev builds (e.g. `1.9.0+custom`) are rejected: pip cannot resolve them from PyPI + when Databricks builds the serverless environment, which would fail every task at runtime. + """ + try: + installed = version("dbt-databricks") + except PackageNotFoundError as exc: + raise RuntimeError( + "dbt-databricks is not installed in the bundle venv — run `make setup` before deploying." + ) from exc + + from packaging.version import InvalidVersion, Version + + try: + parsed = Version(installed) + except InvalidVersion: + parsed = None + if parsed is None or parsed.local or parsed.is_devrelease: + raise RuntimeError( + f"The installed dbt-databricks version ({installed}) is not a plain PyPI release, so the " + "serverless environment cannot install it. Set a released version in pyproject.toml and " + "re-run `make setup` before deploying." + ) + return f"dbt-databricks=={installed}" + + +def _serverless_environment_spec() -> dict: + """Contents of the base-environment file: the environment version plus the pinned + dbt-databricks dependency. + + Only dbt-databricks is listed. The runner notebook (notebook/run_dbt_command.py) also imports + databricks-sdk, but that is a transitive dependency of dbt-databricks, so pinning + dbt-databricks pulls it into the serverless environment automatically — no separate entry + needed. If you ever build a task environment without dbt-databricks, add databricks-sdk here + too, or the runner fails at import.""" + return {"environment_version": "4", "dependencies": [_dbt_databricks_dependency()]} + + +def _write_serverless_environment_file() -> None: + """Write the base-environment file to the bundle root so `bundle deploy` syncs it to the + workspace, where the generated job references it via base_environment. The write is skipped + when the file already has the desired content, so commands like `bundle validate` stay free + of side effects once the file exists.""" + content = yaml.safe_dump(_serverless_environment_spec(), sort_keys=False) + try: + with open(SERVERLESS_ENV_FILE, "r", encoding="utf-8") as f: + if f.read() == content: + return + except OSError: + pass + try: + with open(SERVERLESS_ENV_FILE, "w", encoding="utf-8") as f: + f.write(content) + except OSError as exc: + raise RuntimeError( + f"Cannot write {SERVERLESS_ENV_FILE}, which the generated job needs as its serverless " + f"base environment: {exc}" + ) from exc + + +def build_job(target: str) -> Job: + """Builds the Databricks job that runs the dbt project on serverless via notebook tasks.""" + return Job.from_dict( + { + "name": JOB_NAME, + "queue": {"enabled": True}, + "trigger": { + # Run this job every day, exactly one day from the last run; + # see https://docs.databricks.com/api/workspace/jobs/create#trigger + "periodic": {"interval": 1, "unit": "DAYS"}, + }, + "tasks": _build_tasks(target), + "environments": [ + { + "environment_key": ENVIRONMENT_KEY, + # Pre-built base env (synced with the bundle), not inline deps — + # the two are mutually exclusive; the file pins dbt-databricks. + "spec": {"base_environment": SERVERLESS_ENV_PATH}, + } + ], + } + ) + + +def load_resources(bundle: Bundle) -> Resources: + """ + Called by the Databricks CLI during `bundle deploy` to load resources defined in Python. + + Reads the dbt manifest, writes the serverless base-environment file (synced with the bundle), + and registers the generated job. After deployment this function is not used. + """ + _write_serverless_environment_file() + resources = Resources() + resources.add_job(JOB_NAME, build_job(bundle.target)) + return resources diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/analyses/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/analyses/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__init__.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__version__.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__version__.py new file mode 100644 index 00000000..a8d4557d --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__version__.py @@ -0,0 +1 @@ +__version__ = "0.3.5" diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/dbt_factory.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/dbt_factory.py new file mode 100644 index 00000000..39cc0788 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/dbt_factory.py @@ -0,0 +1,1347 @@ +import heapq +from collections.abc import Iterator +from dataclasses import dataclass, field, replace +from pathlib import PurePosixPath, PureWindowsPath +from typing import cast + +from databricks_dbt_factory.dbt_task import DbtTask +from databricks_dbt_factory.task_factory import TaskFactory, TestTaskFactory +from databricks_dbt_factory.utils import DYNAMIC_VALUE_REFERENCE, build_task_key_maps + +# The `unique_id` prefixes of resources a test can be attached to. +_DBT_TEST_TARGET_PREFIXES = ("model.", "seed.", "snapshot.", "source.") + + +def _flatten_fqn(fqn: list[str]) -> list[str]: + """ + Flattens an fqn the way dbt does before comparing it, splitting every segment on `.`. + + `is_selected_node` treats dots inside a segment as namespace separators, so a test named + `check.nested` with fqn `['probe', 'check.nested']` compares as `['probe', 'check', 'nested']` — + which is why the shorter `probe.check` matches it as a subtree parent. + """ + return [part for segment in fqn for part in segment.split(".")] + + +@dataclass +class _SelectionPlan: + """A dbt selector paired with the indirect-selection mode needed to keep it exact.""" + + select: str + indirect_selection: str + + +@dataclass +class _Gating: + """ + What deciding a node's gating test edges needs, in per-test mode. + + `tests` indexes each test under its referenced resources, `ancestors` records strict dbt ancestors, + `resources_by_task_key` maps immediate emitted dependencies back to manifest resource ids, and + `eligible_test_keys` memoizes the tests eligible at each resource's downstream frontier. + """ + + tests: dict[str, list[tuple[str, frozenset[str]]]] = field(default_factory=dict) + ancestors: dict[str, set[str]] = field(default_factory=dict) + resources_by_task_key: dict[str, str] = field(default_factory=dict) + eligible_test_keys: dict[str, frozenset[str]] = field(default_factory=dict) + + +def _candidate_file_names(original_file_path: str) -> frozenset[str]: + """Returns every base name implied by POSIX and Windows path semantics.""" + return frozenset( + { + PurePosixPath(original_file_path).name, + PureWindowsPath(original_file_path).name, + } + ) + + +def _base_file_name(original_file_path: str) -> str: + """ + Returns the path-flavour-invariant base name dbt's `file:` selector matches. + + dbt assembles `original_file_path` with `os.path.join`/`os.path.relpath` and `FileSelectorMethod` + splits it with `Path(...).name`, so a path containing a backslash can mean either a Windows hierarchy + or a literal POSIX file name. Manifest JSON carries no producer-platform metadata. A `file:` term + is therefore emitted only when both path flavours yield the same base name; otherwise the remaining + selector terms must prove exactness or generation refuses the resource. + """ + candidates = _candidate_file_names(original_file_path) + return next(iter(candidates)) if len(candidates) == 1 else "" + + +class DbtFactory: + """A factory for generating Databricks job definitions from dbt manifests.""" + + def __init__( + self, + task_factories: dict[str, TaskFactory], + bundle_tests: bool = False, + ): + """ + Initializes the dbt factory. + + Args: + task_factories (dict[str, TaskFactory]): Maps dbt resource types (`model`, `seed`, + `snapshot`, `test`) to their respective `TaskFactory` instances. Omitting `test` + disables test-task generation entirely. + bundle_tests (bool): When True, emit one bundled `_test` task per tested + resource and rewire downstream models/seeds/snapshots to depend on the upstream's + bundled test task so failing tests halt the DAG. When False, emit one task per + dbt test node. + """ + self.task_factories = task_factories + self.bundle_tests = bundle_tests + + def create_tasks(self, dbt_manifest: dict) -> list[dict]: + """ + Generates the Databricks task dictionaries from a dbt manifest. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[dict]: Task dictionaries ready to be injected into the `tasks` list of a + Databricks job spec. + + Raises: + ValueError: If the generated job would exceed Databricks' 1,000-task limit. + """ + tasks = self._create_tasks(dbt_manifest) + if len(tasks) > 1_000: + raise ValueError( + f"Databricks jobs support at most 1,000 tasks; this manifest generates {len(tasks):,}. " + f"Reduce the generated resources or enable test bundling." + ) + return [task.to_dict() for task in tasks] + + _GATEABLE_TYPES = frozenset({"model", "seed", "snapshot"}) + _DBT_TEST_TARGET_PREFIXES = _DBT_TEST_TARGET_PREFIXES + + # Characters that still change how an *explicit* `fqn:` selector is interpreted, so a component + # containing one cannot be used to address a node. Each verified against dbt 1.12.0 with `dbt ls`. + # + # ' ' union separator (`graph/cli.py` splits the raw spec on spaces) — the fragments then match + # independently. dbt rejects a space in a resource *name* outright, but a space in a + # *directory* reaches the fqn, and `fqn:probe.my dir.orders` selects nothing. + # ',' intersection separator — the component is read as two, matching zero nodes + # '*?[]' `fnmatch` pattern syntax, still honoured after the `fqn:` prefix: `fqn:probe.a*star` + # selects `a*star` *and* `aXstar`, so the glob is live rather than literal + # + # Databricks substitutes complete `{{...}}` dynamic value references in task commands before the + # task runs. Those references are screened separately; literal single braces and incomplete pairs + # remain ordinary selector text under `fqn:`. + # + # Emitting `fqn:` rather than a bare value is what keeps this list short. dbt's + # `SelectionCriteria.default_method` dispatches a bare value containing `/` to `MethodName.Path` and + # one ending `.sql`/`.py`/`.csv` to `MethodName.File`, both of which then match nothing; naming the + # method explicitly bypasses that heuristic entirely. Verified: bare `probe.orders.sql` and + # `probe.check/slash` select nothing, while `fqn:probe.orders.sql` and `fqn:probe.check/slash` each + # resolve to exactly their node. `:` and literal braces are likewise literal under `fqn:`. + _SELECTOR_METACHARACTERS = frozenset(" ,*?[]") + + @classmethod + def _node_select( + cls, + node_info: dict, + source_info: dict | None = None, + peers: dict | None = None, + ) -> str: + """ + Returns the dbt `--select` argument addressing exactly one node. + + `peers` is every selectable resource in the manifest, keyed by id, used to prove the finished + selector matches only `node_info` (see `_assert_exact`). It is optional so the library API and + unit tests can build a selector for a lone node, but callers with a manifest should pass it: + without it, exactness is assumed rather than established. + + Every node is addressed the same way — the intersection (`,`, dbt's AND) of every independent + fact the manifest gives us about it: + + ,package:,file:[,test_name:] + + No term is exact alone, which is why they are combined rather than chosen between: + + * the **fqn** is matched as a positional path *prefix*, so it also selects nodes nested + beneath it — and dbt compares a node's fqn with its package stripped too, so a package's + `models/probe/alpha.sql` is matched by the root project's `pkg.alpha` as well; + * **`package:`** narrows to the owning package, but matches all of it; + * **`file:`** narrows to one source file, but matches that base name in every package, and a + `schema.yml` is shared by every test declared in it; + * **`test_name:`** narrows a generic test to its type (`not_null`, `unique`), separating tests + that share a `schema.yml` — but only when their types differ, so two `not_null` tests in one + file still need the bare **name**, which dbt matches against the fqn's leaf. The name stands + in for the fqn when the fqn itself is unusable. + + A term is omitted when dbt cannot express it literally or when combining it with the other + terms would form a Databricks dynamic value reference. Exactness is rechecked against `peers` + after any omission. `source_info` addresses a source's tests, which dbt selects by + `source:..
` rather than by fqn. + + **The fqn or the bare name must survive, and the result must then be checked for exactness.** + + dbt has no `unique_id:` selector method, so a selector is always a *predicate*, and its + exactness has to be established rather than assumed. Requiring the fqn or the bare name is + necessary — `package:`, `file:`, `resource_type:` and `test_name:` each address a group — but + is not sufficient. An fqn is a positional prefix over dbt's *flattened* fqn, so it names a + subtree, not a node: + + * a test named `check.nested` flattens to `[probe, check, nested]`, so the sibling `check`'s + selector `probe.check` matches it too — even with `package:`, `file:` and `test_name:` all + present and identical; + * a unit test on `orders` (`[probe, orders, unit_orders]`) and a data test named + `orders.unit_orders` (`[probe, orders.unit_orders]`) flatten identically, which `resource_type:` + does separate; + * dbt clones a unit test per model version leaving both clones the same fqn, name and file; + * two generic tests may simply *share* an fqn: dbt does not require test names to be unique and + disambiguates in the `unique_id` hash only. + + `_assert_exact` checks direct selectors against the manifest. Test selectors whose direct form is + ambiguous may instead use `_test_selection_plan`, which scopes the selector to one exact parent. + + Exactness is checked per node against the rest of the manifest because it is a property of the + node and its selectable neighbours, not of one manifest entry in isolation. + + Raises: + ValueError: when neither the fqn nor the bare name survives, when the final selector cannot + avoid a Databricks dynamic value reference, or when it also matches another resource. + """ + if source_info is not None: + return cls._source_select(source_info) + + # Exactly one of the fqn and the bare name is used, and one of them must be: they are the only + # terms that address a *single* resource, so a selector without either could run another task's + # resource. dbt offers no `unique_id:` method — the full set it accepts, confirmed against dbt + # 1.12.0, is `fqn`, `tag`, `source`, `path`, `file`, `package`, `config`, `test_name`, + # `test_type`, `resource_type`, `state`, `exposure`, `metric`, `result`, `source_status`, + # `group`, `version`, `access`, `semantic_model`, `saved_query`, `unit_test`, `selector`. + terms: list[str] = [] + fqn = node_info.get("fqn") or [] + # The `fqn:` prefix does *not* neutralise graph operators: `fqn:probe.orders+1` still selects + # `orders` and its children, verified with `dbt ls` on dbt 1.12.0. The boundary check therefore + # applies to the joined value; naming the method only bypasses the dispatch heuristic. + if fqn and cls._is_usable_selector(".".join(fqn)) and all(cls._is_usable_component(part) for part in fqn): + # `fqn:` names the method explicitly so `/` and `.sql` remain literal fqn text instead of + # dispatching to path or file matching. `fqn:probe.orders.sql` and + # `fqn:probe.check/slash` each resolve to exactly their node on dbt 1.12.0. + terms.append(f"fqn:{'.'.join(fqn)}") + elif cls._is_usable_component(name := node_info.get("name") or "") and cls._is_usable_selector(name): + # The fqn is unusable, so fall back to the bare resource name, which dbt matches against + # the fqn's leaf. It is the only term that tells apart two nodes sharing a package, a + # file and a test type — two `not_null` tests in one `schema.yml`, say. Not used *with* a + # usable fqn, whose leaf already carries it. + terms.append(f"fqn:{name}") + else: + raise cls._unaddressable(node_info) + + package = node_info.get("package_name") or "" + if cls._is_usable_component(package): + terms.append(f"package:{package}") + + file_name = cls._base_file_name(node_info.get("original_file_path") or "") + if cls._is_usable_component(file_name): + terms.append(f"file:{file_name}") + + # `resource_type:` separates nodes whose flattened fqns coincide across *types*, which the + # other terms cannot: a unit test `unit_orders` on `orders` (fqn [probe, orders, unit_orders]) + # and a data test named `orders.unit_orders` (fqn [probe, orders.unit_orders]) flatten to the + # same string and share their file. Confirmed on dbt 1.12.0: adding `resource_type:unit_test` + # and `resource_type:test` respectively isolates each to one node. + # + # It does *not* undo dbt's indirect selection: `resource_type:test` still admits the attached + # tests eagerly pulled in with a selected model, because dbt intersects sets that are already + # expanded. Suppressing that expansion is the selection plan's job, not this term's: direct test + # plans pin `--indirect-selection empty`, and parent-scoped cautious plans are accepted only when + # `_eager_expansion_superset` proves the expanded result is the intended test alone. + resource_type = node_info.get("resource_type") or "" + if cls._is_usable_component(resource_type): + terms.append(f"resource_type:{resource_type}") + + test_name = (node_info.get("test_metadata") or {}).get("name") or "" + if cls._is_usable_component(test_name): + terms.append(f"test_name:{test_name}") + + select = cls._compose_selector_terms(terms, node_info) + if peers is not None: + cls._assert_exact(select, node_info, peers) + return select + + @classmethod + def _compose_selector_terms(cls, terms: list[str], node_info: dict) -> str: + """Joins terms while preserving the required first term and omitting unsafe optional terms.""" + safe_terms = list(terms) + while match := DYNAMIC_VALUE_REFERENCE.search(select := ",".join(safe_terms)): + offset = 0 + closing_term = 0 + for index, term in enumerate(safe_terms): + offset += len(term) + if match.end() <= offset: + closing_term = index + break + offset += 1 + if closing_term == 0: + raise cls._dynamic_reference(node_info, select) + safe_terms.pop(closing_term) + return select + + @classmethod + def _assert_no_dynamic_reference(cls, select: str, node_info: dict) -> None: + """Rejects a complete dynamic value reference in a final Databricks task selector.""" + if DYNAMIC_VALUE_REFERENCE.search(select): + raise cls._dynamic_reference(node_info, select) + + @classmethod + def _test_selection_plan(cls, test_info: dict, peers: dict) -> _SelectionPlan: + """ + Builds an exact plan for one direct test task. + + An already-exact test selector runs with indirect selection disabled. If the direct selector + matches multiple tests, a test with exactly one manifest parent can be isolated by intersecting + the parent's exact selector with the test selector under `cautious`, provided the conservative + per-term expansion proof shows that the finished selector can run only the intended test. + """ + select = cls._node_select(test_info) + intended = cls._own_ids(test_info, peers) + selected = set(cls._matching_ids(select, peers)) + if selected == intended: + return _SelectionPlan(select, "empty") + + surplus = selected - intended + missing = intended - selected + if missing or not surplus: + cls._assert_exact(select, test_info, peers) + + parents = cls._test_parent_ids(test_info) + if len(parents) != 1: + cls._assert_exact(select, test_info, peers) + parent_id = next(iter(parents)) + if parent_id not in peers: + cls._assert_exact(select, test_info, peers) + parent_info = peers[parent_id] + parent_select = cls._node_select( + parent_info, + source_info=parent_info if parent_info.get("resource_type") == "source" else None, + peers=peers, + ) + scoped_select = f"{parent_select},{select}" + cls._assert_no_dynamic_reference(scoped_select, test_info) + if cls._eager_expansion_superset(scoped_select, peers) != intended: + cls._assert_exact(select, test_info, peers) + return _SelectionPlan(scoped_select, "cautious") + + @classmethod + def _eager_expansion_superset(cls, select: str, peers: dict) -> set[str]: + """ + Returns a conservative superset of a cautious selector's result. + + dbt expands every comma-separated term before intersecting the term results. For each term this + proof uses eager expansion: its direct matches plus every test with any directly matched parent. + Cautious expansion can only admit fewer tests, so intersecting these eager term sets remains a + superset of the finished cautious result. A parent-scoped plan is accepted only when that superset + is exactly the intended test. + + The intended single-parent test is present by construction: every test term matches it directly, + and every exact-parent term matches its sole parent and therefore adds the test indirectly. + """ + possible: set[str] | None = None + for term in select.split(","): + direct = set(cls._matching_ids(term, peers)) + expanded = direct | cls._tests_attached_to_any(direct, peers) + possible = expanded if possible is None else possible & expanded + return possible or set() + + @classmethod + def _tests_attached_to_any(cls, parent_ids: set[str], peers: dict) -> set[str]: + """Returns enabled test ids having at least one manifest parent in `parent_ids`.""" + if isinstance(peers, _SelectorIndex): + return peers.tests_attached_to_any(parent_ids) + return { + full_name + for full_name, info in peers.items() + if info.get("resource_type") in {"test", "unit_test"} and cls._test_parent_ids(info) & parent_ids + } + + @classmethod + def _test_parent_ids(cls, test_info: dict) -> frozenset[str]: + """Returns every model, seed, snapshot, or source dependency id, including absent resources.""" + return frozenset( + dep + for dep in test_info.get("depends_on", {}).get("nodes", []) + if dep.startswith(cls._DBT_TEST_TARGET_PREFIXES) + ) + + @classmethod + def _assert_exact(cls, select: str, node_info: dict, peers: dict) -> None: + """ + Raises unless `select` runs only the node it was built for. + + Direct test plans pin `--indirect-selection empty`, so their exactness is a plain intersection of + the emitted terms. Parent-scoped cautious plans apply their additional proof in + `_test_selection_plan`. Bundled tasks reuse these per-test plans and union only plans with the + same indirect-selection mode. + + The contract is **equality**, not "no surplus". A selector that matches nothing is just as wrong as + one that matches too much: `dbt test` and `dbt run` both exit 0 on a zero-match selector, so the + task would go green having asserted or built nothing. + + The node recognises itself by object identity *or* by `unique_id`: identity alone would call a + node its own collision if a caller passed a copy, and `unique_id` alone fails on hand-written + fixtures that omit the field. + """ + allowed = cls._own_ids(node_info, peers) + run = set(cls._matching_ids(select, peers)) + surplus = run - allowed + if surplus: + raise cls._ambiguous(node_info, select, sorted(surplus)) + missing = allowed - run + if missing: + raise cls._selects_nothing(node_info, select, sorted(missing)) + + @staticmethod + def _own_ids(node_info: dict, peers: dict) -> set[str]: + """ + The `peers` keys that *are* this node, by `unique_id` when present or by object identity. + + `unique_id` is a direct dict lookup, so the common path costs nothing. The identity scan is the + fallback for hand-written fixtures that omit the field — walking every peer here once per node was + itself quadratic, and defeated the index added to avoid exactly that. + """ + own_id = node_info.get("unique_id") + if own_id is not None and peers.get(own_id) is not None: + return {own_id} + return {full_name for full_name, info in peers.items() if info is node_info} + + @staticmethod + def _base_file_name(original_file_path: str) -> str: + """The base name dbt's `file:` selector matches — see the module-level `_base_file_name`.""" + return _base_file_name(original_file_path) + + @classmethod + def _source_select(cls, source_info: dict) -> str: + """ + Returns the `source:..
` selector for a source's tests. + + A source has no fqn selector of its own, so all three parts must be usable: unlike a node + there is no other term to fall back on. + + `.` is disallowed on top of the usual metacharacters. It delimits this grammar, which accepts + at most three parts, so a dot inside one makes four and dbt rejects the whole selector with a + Runtime Error ("Invalid source selector value") rather than merely selecting nothing — + confirmed against dbt 1.12.0 with `dbt ls`. A node's fqn is unaffected: there the dot is the + separator we are already building with, and a dotted segment stays addressable. + + Complete Databricks dynamic value references are checked after assembly, including references + completed by the separator between the source and table. None of the three source parts can be + omitted, so such a selector is refused with the unsafe final value in the error. + + The finished string is checked with `_is_usable_selector` too: like a bare name, the whole + `source:...` is one raw selector, so a trailing `+N` on the table is read as a graph operator + and matches nothing while `dbt test` still exits 0. Only the boundary matters — + `source:pkg.raw.2+ord` resolves exactly — so the check goes on the assembled string rather + than on each part. + """ + package = source_info.get("package_name") or "" + source_name = source_info.get("source_name") or "" + table = source_info.get("name") or "" + select = f"source:{package}.{source_name}.{table}" + cls._assert_no_dynamic_reference(select, source_info) + parts = (package, source_name, table) + if not all(cls._is_usable_component(part) and "." not in part for part in parts): + raise cls._unaddressable(source_info) + if not cls._is_usable_selector(select): + raise cls._unaddressable(source_info) + return select + + @classmethod + def _is_usable_component(cls, value: str) -> bool: + """Whether `value` can appear inside a selector component and still mean itself.""" + return ( + bool(value) + and not cls._SELECTOR_METACHARACTERS & set(value) + and DYNAMIC_VALUE_REFERENCE.search(value) is None + ) + + @classmethod + def _is_usable_selector(cls, value: str) -> bool: + """ + Whether a fully assembled selector avoids dbt graph expansion and Databricks substitution. + + dbt reads a trailing `+N` as child depth even after an explicit method prefix, so + `fqn:probe.orders+1` selects `orders` and its child. Leading `@` and `N+` forms, and `+` inside + a segment, remain literal under an explicit `fqn:` method; `fqn:@weird`, `fqn:2+orders`, + `fqn:pkg.+leading`, and `fqn:pkg.raw.2+ord` each address their exact node on dbt 1.12.0. + + Complete `{{...}}` references are unusable because Databricks substitutes them before dbt receives + the command. The check applies after selector components are joined so references spanning + components are also rejected. + """ + return not value.rstrip("0123456789").endswith("+") and DYNAMIC_VALUE_REFERENCE.search(value) is None + + @staticmethod + def _flat_fqn(fqn: list[str]) -> list[str]: + """Flattens an fqn as dbt does — see the module-level `_flatten_fqn`.""" + return _flatten_fqn(fqn) + + @classmethod + def _fqn_term_matches(cls, term: str, node_info: dict) -> bool: + """ + Whether dbt's fqn selector `term` matches `node_info`. + + Mirrors `QualifiedNameSelectorMethod.node_is_match` on dbt 1.12.0, including the two details + that make an fqn a *predicate* rather than an identifier: + + * the comparison is a positional walk over the *flattened* fqn, and a selector that runs out of + parts still matches — so `probe.check` matches `['probe', 'check', 'nested']`, i.e. anything + nested beneath it, whether the nesting comes from a directory or from a dot in a name; + * it is retried with the node's own package stripped, so a package's `models/probe/alpha.sql` + is matched by the root project's `probe.alpha` too. + + Wildcards are not handled: `_is_usable_component` rejects `*?[]` outright, so no term reaching + here contains one. This mirrors dbt rather than reasoning about it because being *lenient* here + would mean accepting a selector that matches two nodes, which is the whole bug class; the + integration suite pins the mirror against `dbt ls`. + """ + fqn = node_info.get("fqn") or [] + if not fqn: + # A manifest missing `fqn` is not something dbt produces, but a hand-rolled or truncated one + # can be, and the selector then falls back to the bare name. dbt would still match that name + # against the fqn's leaf, so compare against the name rather than declining outright — + # otherwise the exactness check reports a selector that reaches nothing. + return bool(node_info.get("name")) and node_info.get("name") == term + # dbt's `is_versioned` requires the resource type to be a model (`VERSIONED_NODE_TYPES`), not + # merely that `version` is set — and a unit-test clone *does* carry `version`. Deriving it from + # the field alone takes dbt's versioned branch for unit tests and so skips its plain + # `fqn[-1] == term` match, making the mirror stricter than dbt and hiding a real collision on + # such a node's bare-name selector. + is_versioned = node_info.get("resource_type") == "model" and node_info.get("version") is not None + return cls._is_selected_node(fqn, term, is_versioned) or cls._is_selected_node(fqn[1:], term, is_versioned) + + @classmethod + def _is_selected_node(cls, fqn: list[str], term: str, is_versioned: bool) -> bool: + """Mirrors dbt's `is_selected_node` for the operator-free, wildcard-free terms we emit.""" + if not fqn: + return False + selector_parts = term.split(".") + if cls._matches_fqn_leaf(fqn, term, selector_parts, is_versioned): + return True + flat_fqn = cls._flat_fqn(fqn) + if len(flat_fqn) < len(selector_parts): + return False + for index, part in enumerate(selector_parts): + if flat_fqn[index] != part: + return False + return True + + @staticmethod + def _matches_fqn_leaf(fqn: list[str], term: str, selector_parts: list[str], is_versioned: bool) -> bool: + """ + Whether `term` matches the fqn's leaf, dbt's shortcut before the positional walk. + + A versioned model's fqn ends `[..., , v]`, and dbt lets the last two segments match on + either the `.` or `_` delimiter, so `orders`, `orders.v1` and `orders_v1` all hit + `['probe', 'orders', 'v1']`. + """ + if not is_versioned: + return fqn[-1] == term + if len(fqn) < 2: + return False + return fqn[-2] == term or "_".join(fqn[-2:]) == "_".join(selector_parts[-2:]) + + @classmethod + def _term_matches(cls, term: str, node_info: dict) -> bool: + """Whether one emitted selector term matches `node_info`.""" + method, _, value = term.partition(":") + if not _ or method == "fqn": + # A bare value is still accepted so a hand-written selector keeps working; everything the + # factory emits now carries the explicit `fqn:` method. + return cls._fqn_term_matches(value if method == "fqn" else term, node_info) + if method == "source": + parts = value.split(".") + return ( + len(parts) == 3 + and node_info.get("resource_type") == "source" + and (node_info.get("package_name") or "") == parts[0] + and (node_info.get("source_name") or "") == parts[1] + and (node_info.get("name") or "") == parts[2] + ) + if method == "package": + return (node_info.get("package_name") or "") == value + if method == "file": + # dbt's `FileSelectorMethod` matches the base name *or* its stem, so `file:a.yml` also + # matches a node declared in `a.yml.yml`. Mirroring only the name would let that collision + # past `_assert_exact` — confirmed with `dbt ls` on dbt 1.12.0, where the `a.yml` task's + # selector resolves to the `a.yml.yml` test as well. + for base in _candidate_file_names(node_info.get("original_file_path") or ""): + if value in (base, base.rsplit(".", 1)[0] if "." in base else base): + return True + return False + if method == "resource_type": + return (node_info.get("resource_type") or "") == value + if method == "test_name": + return ((node_info.get("test_metadata") or {}).get("name") or "") == value + return True # pragma: no cover - no other method is emitted + + @staticmethod + def _selector_index(peers: dict) -> dict: + """Wraps `peers` in the narrowing index used for exactness checks (see `_SelectorIndex`).""" + return _SelectorIndex(peers) + + @classmethod + def _matching_ids(cls, select: str, candidates: dict) -> list[str]: + """ + The ids in `candidates` that `select` matches, by intersecting every term as dbt does. + + This is a *direct* match: it does not model dbt's indirect selection, so it is only exactness + evidence for a plan that suppresses expansion. Direct test plans pin `--indirect-selection empty` + and so need nothing further; parent-scoped cautious plans are proved instead by + `_eager_expansion_superset`, which expands each term itself. + + `candidates` should be a `_SelectorIndex` on any real manifest: scanning every node for every + node is quadratic, and measurably so — 90 seconds for a 6,000-node manifest. The index narrows the + scan to the nodes sharing this selector's `package:` and `file:`, which is a handful. A plain dict + still works, for the unit tests and library callers that pass one. + """ + terms = select.split(",") + scan = candidates.narrow(terms) if isinstance(candidates, _SelectorIndex) else candidates + matched: list[str] = [] + for full_name, info in scan.items(): + if all(cls._term_matches(term, info) for term in terms): + matched.append(full_name) + return matched + + @staticmethod + def _ambiguous(node_info: dict, select: str, also_matched: list[str]) -> ValueError: + """ + Builds the error raised when a selector is valid but not exact. + + Distinct from `_unaddressable`: nothing about *this* resource's name is wrong, so the remedy is + about the collision with its neighbours rather than about selector syntax. + """ + name = node_info.get("name") + path = node_info.get("original_file_path") + return ValueError( + f"Cannot generate a task for {name!r} ({path}): the generated selector for it " + f"({select}) also runs {', '.join(sorted(also_matched))}. dbt has no unique-id selector, so " + f"the factory cannot prove a task addresses these separately. It does not search alternate " + f"spellings once a usable FQN exists, and refuses to risk running the others before their own " + f"dependencies have completed. Rename {name!r} so that its dotted " + f"name neither matches nor prefixes a " + f"sibling's, or move it to a file of its own." + ) + + @staticmethod + def _dynamic_reference(node_info: dict, select: str) -> ValueError: + """Builds the error for a selector Databricks would interpret as a dynamic value reference.""" + name = node_info.get("name") + path = node_info.get("original_file_path") + return ValueError( + f"Cannot generate a task for {name!r} ({path}): the final selector ({select}) contains a " + f"Databricks dynamic value reference. Rename the resource or file so its selector terms do " + f"not form a complete reference." + ) + + @staticmethod + def _selects_nothing(node_info: dict, select: str, missing: list[str]) -> ValueError: + """ + Builds the error raised when a selector fails to reach what the task is meant to run. + + Distinct from `_ambiguous`, which is the opposite failure. This one is easy to overlook because it + is silent at run time: `dbt test` and `dbt run` exit 0 on a selector that matches nothing, so the + task goes green having asserted or built nothing at all. + """ + name = node_info.get("name") + path = node_info.get("original_file_path") + return ValueError( + f"Cannot generate a task for {name!r} ({path}): the selector dbt offers for it ({select}) " + f"does not reach {', '.join(missing)}. dbt exits 0 for a selector that matches nothing, so " + f"the task would report success having run nothing. This is a bug in selector construction " + f"rather than something to fix in the project — please report it." + ) + + @staticmethod + def _unaddressable(node_info: dict) -> ValueError: + """ + Builds the error raised when no selector can address a node. + + This message is the whole of what a CLI user sees (`main` reports it without a traceback), so + it leads with the resource and the remedy and keeps the reasoning to one closing line. + """ + name = node_info.get("name") + path = node_info.get("original_file_path") + return ValueError( + f"Cannot generate a task for {name!r} ({path}): dbt cannot select it uniquely. " + f"Rename the resource or its file so that it does not end with a dbt graph operator (a " + f"trailing +N), contains none of a space, comma or one of *?[], and contains no complete " + f"Databricks dynamic value reference (`{{{{...}}}}`) — or, for a source, a dot. Without a " + f"usable name or path, the only terms left match a group of " + f"resources, so the task could run another task's resource." + ) + + def _create_tasks(self, dbt_manifest: dict) -> list[DbtTask]: + """ + Builds `DbtTask` instances from the manifest, applying the bundling and gating policies. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[DbtTask]: `DbtTask` instances (not yet rendered to dicts). + """ + dbt_nodes = self._enabled_only(dbt_manifest.get("nodes", {})) + dbt_sources = self._enabled_only(dbt_manifest.get("sources", {})) + dbt_unit_tests = self._enabled_only(dbt_manifest.get("unit_tests", {})) + + # All directly selectable resources participate in selector exactness and test selection-plan + # checks, including sources even though no emitted command builds a source directly. + peers = _SelectorIndex({**dbt_nodes, **dbt_unit_tests, **dbt_sources}) + + bundle = "test" in self.task_factories and self.bundle_tests + bundled_tests: dict[str, list[tuple[str, dict]]] = {} + standalone_tests: list[tuple[str, dict]] = [] + if bundle: + bundled_tests, standalone_tests = self._classify_tests(dbt_nodes, dbt_sources, dbt_unit_tests) + standalone_test_ids = {full_name for full_name, _ in standalone_tests} + + # Unit tests live under the manifest `unit_tests` key, not `nodes`. In per-test mode each + # gets its own task (those whose target model is absent are skipped), so include their ids + # here to receive a task key from `build_task_key_maps`. + unit_test_ids = ( + self._emitted_unit_test_ids(dbt_unit_tests, dbt_nodes) + if not bundle and "test" in self.task_factories + else [] + ) + + task_ids = [] + for full_name, info in dbt_nodes.items(): + if self._node_gets_own_task(full_name, info, bundle, standalone_test_ids): + task_ids.append(full_name) + task_ids += unit_test_ids + task_keys, bundled_test_keys = build_task_key_maps(task_ids, sorted(bundled_tests)) + + gating = _Gating() + if not bundle and "test" in self.task_factories: + indexed_tests = self._index_tests_by_resource(dbt_nodes, dbt_sources, dbt_unit_tests, task_keys) + gating = _Gating( + tests=indexed_tests, + ancestors=self._compute_ancestors(dbt_nodes, dbt_sources) if indexed_tests else {}, + resources_by_task_key={task_key: full_name for full_name, task_key in task_keys.items()}, + ) + + tasks = self._build_resource_tasks( + dbt_nodes, + bundle, + task_keys, + bundled_test_keys, + gating, + peers, + ) + + if bundle: + tasks.extend( + self._build_bundled_test_tasks( + dbt_nodes, + dbt_sources, + bundled_tests, + task_keys, + bundled_test_keys, + peers, + ) + ) + tasks.extend(self._build_standalone_test_tasks(standalone_tests, task_keys, peers)) + elif "test" in self.task_factories: + tasks.extend(self._build_unit_test_tasks(dbt_unit_tests, task_keys, peers)) + + return tasks + + @staticmethod + def _enabled_only(entries: dict) -> dict: + """ + Drops entries dbt has disabled. + + dbt normally files an `enabled=false` resource under the manifest's own `disabled` key, which + we never read — but not always: a versioned model whose declared version has no file leaves + its test in `nodes` with `config.enabled` false. dbt selects nothing for such a node, and + `dbt test` on a zero-match selector still exits 0, so a task built from it would go green + having asserted nothing. Filtering here rather than at each decision site keeps the task-key + map, the bundling classification and the dependency graph working from one view of the + manifest. Confirmed against dbt 1.12.0. + """ + return { + full_name: info + for full_name, info in entries.items() + if (info.get("config") or {}).get("enabled") is not False + } + + def _node_gets_own_task(self, full_name: str, node_info: dict, bundle: bool, standalone_test_ids: set[str]) -> bool: + """ + Whether a `dbt_nodes` entry becomes its own task (and so receives a task key). True for any + resource type with a factory, except single-resource test nodes in bundle mode — those fold + into their resource's bundled test task. The single authority for this decision, so the + task-key map and the task-building loops stay in agreement. + """ + resource_type = node_info["resource_type"] + if resource_type not in self.task_factories: + return False + if bundle and resource_type == "test" and full_name not in standalone_test_ids: + return False + return True + + def _emitted_unit_test_ids(self, dbt_unit_tests: dict, dbt_nodes: dict) -> list[str]: + """ + Full names of the unit tests that become their own task in per-test mode: those whose + target model resolves and is present in the manifest. This is the emission decision for + unit tests — the returned ids enter `task_ids`, so a unit test's presence in `task_keys` + is how every consumer knows it was emitted. + + Each versioned unit-test clone receives its own task and depends on its exact model version. + """ + emitted: list[str] = [] + for full_name, info in dbt_unit_tests.items(): + model_full_name = self._unit_test_model(info) + if model_full_name is not None and model_full_name in dbt_nodes: + emitted.append(full_name) + return emitted + + def _compute_ancestors(self, dbt_nodes: dict, dbt_sources: dict) -> dict[str, set[str]]: + """ + Maps each testable resource's full name to the set of resources it transitively depends + on (not including itself). Used in per-test mode to decide whether a test can safely + gate a downstream node: a test `T` with refs `R` is only safe to add to node `N`'s + deps if `R ⊆ ancestors(N)` — i.e. `N` already waits for all of `T`'s endpoints, + transitively. Otherwise adding `T` would create a cycle (since `T` depends on each + ref, and some ref might depend on `N`). + """ + resources = {**dbt_nodes, **dbt_sources} + dependencies: dict[str, set[str]] = {} + dependents: dict[str, list[str]] = {full_name: [] for full_name in resources} + for full_name, info in resources.items(): + direct_dependencies = { + dependency for dependency in info.get("depends_on", {}).get("nodes", []) if dependency in resources + } + dependencies[full_name] = direct_dependencies + for dependency in direct_dependencies: + dependents[dependency].append(full_name) + + unresolved_counts = { + full_name: len(direct_dependencies) for full_name, direct_dependencies in dependencies.items() + } + ready = [full_name for full_name, count in unresolved_counts.items() if count == 0] + heapq.heapify(ready) + ancestors: dict[str, set[str]] = {} + while ready: + full_name = heapq.heappop(ready) + node_ancestors: set[str] = set() + for dependency in sorted(dependencies[full_name]): + node_ancestors.add(dependency) + node_ancestors.update(ancestors[dependency]) + ancestors[full_name] = node_ancestors + + for dependent in sorted(dependents[full_name]): + unresolved_counts[dependent] -= 1 + if unresolved_counts[dependent] == 0: + heapq.heappush(ready, dependent) + + if len(ancestors) != len(resources): + unresolved = {full_name for full_name, count in unresolved_counts.items() if count > 0} + cycle = self._dependency_cycle(dependencies, unresolved) + raise ValueError( + f"Cannot compute test gates because the manifest contains the dependency cycle " + f"{' -> '.join(cycle)}. Regenerate the manifest after removing the cycle." + ) + return ancestors + + @staticmethod + def _dependency_cycle(dependencies: dict[str, set[str]], candidates: set[str]) -> list[str]: + """Returns one deterministic cycle from a graph that could not be topologically ordered.""" + active = 1 + complete = 2 + state: dict[str, int] = {} + for start in sorted(candidates): + if state.get(start) == complete: + continue + + path = [start] + positions = {start: 0} + state[start] = active + stack: list[tuple[str, Iterator[str]]] = [(start, iter(sorted(dependencies[start] & candidates)))] + while stack: + full_name, direct_dependencies = stack[-1] + try: + dependency = next(direct_dependencies) + except StopIteration: + stack.pop() + path.pop() + positions.pop(full_name) + state[full_name] = complete + continue + + dependency_state = state.get(dependency) + if dependency_state is None: + positions[dependency] = len(path) + path.append(dependency) + state[dependency] = active + stack.append((dependency, iter(sorted(dependencies[dependency] & candidates)))) + elif dependency_state == active: + return path[positions[dependency] :] + [dependency] + + raise RuntimeError("A non-topological dependency graph did not contain a cycle.") + + def _index_tests_by_resource( + self, dbt_nodes: dict, dbt_sources: dict, dbt_unit_tests: dict, task_keys: dict[str, str] + ) -> dict[str, list[tuple[str, frozenset[str]]]]: + """ + Maps each testable resource's full name to a list of (test_task_key, test_refs) pairs + for every emitted data test. A test's task exit status decides whether the dependency blocks: + warn-severity tests normally succeed, while `--warn-error` can make the same task fail. Keeping + one graph rule preserves dbt's ordering without reproducing its option-dependent severity logic. + + Unit tests are indexed too. A unit test has no severity — it always fails the run — so it + always gates. Only unit tests that were emitted as tasks (present in `task_keys`) are + indexed, so a downstream node never gates on a unit-test task that was skipped. + + The refs set is carried alongside each test so `_extend_deps_with_upstream_tests` can + avoid cycles: a test with refs that aren't all ancestors of a candidate node would + create a cycle if added as that node's dep. + """ + index: dict[str, list[tuple[str, frozenset[str]]]] = {} + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + if node_full_name in task_keys: + self._index_test(index, task_keys[node_full_name], node_info, dbt_nodes, dbt_sources) + + for unit_test_full_name, unit_test_info in dbt_unit_tests.items(): + if unit_test_full_name in task_keys: + self._index_test(index, task_keys[unit_test_full_name], unit_test_info, dbt_nodes, dbt_sources) + return index + + def _index_test( + self, + index: dict[str, list[tuple[str, frozenset[str]]]], + test_task_key: str, + test_info: dict, + dbt_nodes: dict, + dbt_sources: dict, + ) -> None: + """Indexes a test (data or unit) under each resource it references, carrying its ref set.""" + refs = self._testable_refs(test_info, dbt_nodes, dbt_sources) + for resource_full in refs: + index.setdefault(resource_full, []).append((test_task_key, refs)) + + def _testable_refs(self, test_info: dict, dbt_nodes: dict, dbt_sources: dict) -> frozenset[str]: + """Returns the models/seeds/snapshots/sources a test references, as present in the manifest.""" + refs: set[str] = set() + for dep in test_info.get("depends_on", {}).get("nodes", []): + if dep.startswith(self._DBT_TEST_TARGET_PREFIXES) and (dep in dbt_nodes or dep in dbt_sources): + refs.add(dep) + return frozenset(refs) + + @staticmethod + def _unit_test_model(unit_test_info: dict) -> str | None: + """ + Returns the full name of the model a unit test targets, or None if it can't be resolved. + + Prefers dbt's already-resolved `depends_on.nodes[0]`, which is the authority: dbt resolves + the `model` field to a node id at parse time and rewrites unit tests on *versioned* models + to target `model...v` while leaving `model` as the bare name. Rebuilding + `model..` from the `model` field would therefore miss versioned models + entirely (and dbt's ` ` spelling of the field), silently dropping the unit + test. Falls back to that reconstruction only when `depends_on` is absent, so manifests + that predate it keep working. + """ + for dep in unit_test_info.get("depends_on", {}).get("nodes", []): + if dep.startswith("model."): + return dep + model = unit_test_info.get("model") + package = unit_test_info.get("package_name") + if model and package: + return f"model.{package}.{model}" + return None + + @classmethod + def _extend_deps_with_upstream_tests( + cls, + node_full_name: str, + existing_deps: list[str] | None, + gating: _Gating, + ) -> list[str]: + """ + Adds safe test gates at the first downstream frontier. + + A test is eligible for node `N` when all of its refs are strict ancestors of `N`. Tests already + eligible at an immediate emitted dependency are inherited through that dependency and need not be + repeated. The remaining tests are the first frontier where the full ref set has become available. + """ + deps: list[str] = list(existing_deps or []) + eligible = cls._eligible_test_keys(node_full_name, gating) + inherited: set[str] = set() + for dependency_key in deps: + dependency = gating.resources_by_task_key.get(dependency_key) + if dependency is not None: + inherited.update(cls._eligible_test_keys(dependency, gating)) + deps.extend(sorted(eligible - inherited - set(deps))) + return deps + + @staticmethod + def _eligible_test_keys(node_full_name: str, gating: _Gating) -> frozenset[str]: + """Returns and caches tests whose complete ref set is among the node's strict ancestors.""" + cached = gating.eligible_test_keys.get(node_full_name) + if cached is not None: + return cached + + ancestors = gating.ancestors.get(node_full_name, set()) + eligible: set[str] = set() + for ancestor in ancestors: + for test_key, test_refs in gating.tests.get(ancestor, []): + if test_refs <= ancestors: + eligible.add(test_key) + result = frozenset(eligible) + gating.eligible_test_keys[node_full_name] = result + return result + + def _classify_tests( + self, dbt_nodes: dict, dbt_sources: dict, dbt_unit_tests: dict + ) -> tuple[dict[str, list[tuple[str, dict]]], list[tuple[str, dict]]]: + """ + Classifies test nodes for bundled mode so that no test is silently dropped. + + - Tests with exactly 1 testable dep: added by id to that resource's bundled + `_test` task. Each receives its own exact selection plan before the plans are + unioned by indirect-selection mode. + - Tests with >1 testable deps (cross-resource, e.g. `relationships`): emitted as their own + tasks with every available model, seed, or snapshot dependency. + - Tests with 0 testable deps (singular/custom tests that don't `ref()` or `source()` + any resource): also emitted as their own tasks, since no bundle would pick them up. + + Unit tests are assigned explicitly to their resolved model bundle. A model with only unit + tests therefore still gets a bundled task without relying on indirect selection to discover it. + + Returns: + (bundled_tests, standalone_tests): + - `bundled_tests`: exact data and unit test nodes grouped by the resource whose + bundled task runs them. + - `standalone_tests`: list of `(test_full_name, test_node_info)` for tests + that must run as individual tasks (cross-model or zero-dep). + """ + bundled_tests: dict[str, list[tuple[str, dict]]] = {} + standalone_tests: list[tuple[str, dict]] = [] + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + testable_deps = self._testable_refs(node_info, dbt_nodes, dbt_sources) + if len(testable_deps) == 1: + resource_id = next(iter(testable_deps)) + bundled_tests.setdefault(resource_id, []).append((node_full_name, node_info)) + else: + standalone_tests.append((node_full_name, node_info)) + + for unit_test_full_name, unit_test_info in dbt_unit_tests.items(): + model_full_name = self._unit_test_model(unit_test_info) + if model_full_name is not None and model_full_name in dbt_nodes: + bundled_tests.setdefault(model_full_name, []).append((unit_test_full_name, unit_test_info)) + return bundled_tests, standalone_tests + + def _build_resource_tasks( + self, + dbt_nodes: dict, + bundle: bool, + task_keys: dict[str, str], + bundled_test_keys: dict[str, str], + gating: _Gating, + peers: dict, + ) -> list[DbtTask]: + """Builds tasks for every non-test resource, plus per-test tasks when not bundling.""" + # A tested resource resolves to its bundle; sources gain a scheduled key only through this map. + dependency_task_keys = {**task_keys, **bundled_test_keys} if bundle else task_keys + tasks: list[DbtTask] = [] + for node_full_name, node_info in dbt_nodes.items(): + if node_full_name not in task_keys: + continue + if bundle and node_info["resource_type"] == "test": + # Standalone tests are keyed but built by `_build_standalone_test_tasks`, not here. + continue + + resource_type = node_info["resource_type"] + task_key = task_keys[node_full_name] + factory = self.task_factories[resource_type] + if resource_type == "test": + plan = self._test_selection_plan(node_info, peers) + task = cast(TestTaskFactory, factory).create_task( + plan.select, + node_info["name"], + node_info, + task_key, + task_keys, + indirect_selection=plan.indirect_selection, + ) + else: + task = factory.create_task( + self._node_select(node_info, peers=peers), + node_info["name"], + node_info, + task_key, + dependency_task_keys, + ) + + if resource_type in self._GATEABLE_TYPES and not bundle: + task = self._gate_task(task, node_full_name, gating) + tasks.append(task) + return tasks + + def _gate_task( + self, + task: DbtTask, + node_full_name: str, + gating: _Gating, + ) -> DbtTask: + """Adds per-test-mode gates at the first downstream frontier.""" + if not gating.tests: + return task + deps = self._extend_deps_with_upstream_tests(node_full_name, task.depends_on, gating) + return replace(task, depends_on=deps) + + def _build_bundled_test_tasks( + self, + dbt_nodes: dict, + dbt_sources: dict, + bundled_tests: dict[str, list[tuple[str, dict]]], + task_keys: dict[str, str], + bundled_test_keys: dict[str, str], + peers: dict, + ) -> list[DbtTask]: + """Emits one task per tested resource from unions of exact per-test selection plans.""" + test_factory = cast(TestTaskFactory, self.task_factories["test"]) + tasks: list[DbtTask] = [] + for full_name, tests in sorted(bundled_tests.items()): + is_source = full_name.startswith("source.") + info = dbt_sources[full_name] if is_source else dbt_nodes[full_name] + # A source never gets a task of its own, so it has no gate. Any other parent must, or the + # bundle would run its tests with nothing ordering them after the resource was built. Raise + # rather than emit the ungated task: `resource_type` having no registered factory is a + # supported library configuration (see `_node_gets_own_task`), and a `KeyError` here would + # escape `main`'s handler as a traceback that names no resource. + if not is_source and full_name not in task_keys: + raise ValueError( + f"Cannot bundle the tests of {full_name!r}: no task was generated for it, so the " + f"bundled test task would run unordered against it. Register a task factory for " + f"{info.get('resource_type')!r} resources, or generate one task per test instead of " + f"bundling." + ) + tasks.append( + test_factory.create_bundled_task( + task_key=bundled_test_keys[full_name], + selects_by_indirect_selection=self._bundled_selects_by_mode(tests, peers), + deps_command_name=info["name"], + depends_on=[] if is_source else [task_keys[full_name]], + ) + ) + return tasks + + @classmethod + def _bundled_selects_by_mode(cls, tests: list[tuple[str, dict]], peers: dict) -> dict[str, list[str]]: + """Builds and validates the deterministic selector groups for one test bundle.""" + selects_by_mode: dict[str, list[str]] = {} + test_info_by_mode: dict[str, dict] = {} + for _, test_info in sorted(tests, key=lambda item: item[0]): + plan = cls._test_selection_plan(test_info, peers) + selects_by_mode.setdefault(plan.indirect_selection, []).append(plan.select) + test_info_by_mode.setdefault(plan.indirect_selection, test_info) + for mode, selects in selects_by_mode.items(): + cls._assert_no_dynamic_reference(" ".join(sorted(selects)), test_info_by_mode[mode]) + return selects_by_mode + + def _build_standalone_test_tasks( + self, + standalone_tests: list[tuple[str, dict]], + task_keys: dict[str, str], + peers: dict, + ) -> list[DbtTask]: + """ + Emits one task per standalone test — cross-resource tests (e.g. `relationships`) gated on + every referenced model, seed, or snapshot task, plus zero-dep singular tests that bundles + cannot cover. + """ + test_factory = cast(TestTaskFactory, self.task_factories["test"]) + tasks: list[DbtTask] = [] + for test_full_name, test_info in sorted(standalone_tests, key=lambda item: item[0]): + test_task_key = task_keys[test_full_name] + plan = self._test_selection_plan(test_info, peers) + tasks.append( + test_factory.create_task( + plan.select, + test_info["name"], + test_info, + test_task_key, + task_keys, + indirect_selection=plan.indirect_selection, + ) + ) + return tasks + + def _build_unit_test_tasks( + self, + dbt_unit_tests: dict, + task_keys: dict[str, str], + peers: dict, + ) -> list[DbtTask]: + """ + Emits one task per unit test, selected by its full FQN and gated on the model it tests. + Only unit tests that received a task key (see `_emitted_unit_test_ids`) are emitted; unit + tests whose target model is absent from the manifest were never keyed and are skipped, so + their task can't gate on a model task that is never created. Used in per-test mode; in + bundled mode unit tests are assigned explicitly to their model's exact-selector bundle. + + Versioned unit-test clones receive independent parent-scoped selection plans, so each task waits + only for and runs only against its exact model version. + """ + test_factory = cast(TestTaskFactory, self.task_factories["test"]) + + tasks: list[DbtTask] = [] + for unit_test_full_name, unit_test_info in sorted(dbt_unit_tests.items()): + if unit_test_full_name not in task_keys: + continue + plan = self._test_selection_plan(unit_test_info, peers) + tasks.append( + test_factory.create_task( + plan.select, + unit_test_info["name"], + unit_test_info, + task_keys[unit_test_full_name], + task_keys, + indirect_selection=plan.indirect_selection, + ) + ) + return tasks + + +class _SelectorIndex(dict): + """ + The manifest's selectable resources, plus indexes for narrowing an exactness check. + + `_assert_exact` runs once per node, so evaluating each selector against every node makes generation + quadratic — measured at 90 seconds for a 6,000-node manifest, and real projects are larger. Each + index maps a term value to the nodes it *could* match, so a check scans a handful of candidates + instead of the manifest. + + Every bucket is a sound **superset** of the true matches: `_matching_ids` still evaluates the full + predicate on whatever comes back, so narrowing can only cost time, never change the answer. + `test_selector_index_narrowing_matches_a_full_scan` pins that equivalence against a full scan. + + Subclasses `dict` so it *is* the peers mapping: callers that only iterate or look up by id need not + know the indexes exist. + """ + + def __init__(self, peers: dict): + super().__init__(peers) + self._by_package: dict[str, dict] = {} + self._by_file: dict[str, dict] = {} + self._by_test_name: dict[str, dict] = {} + self._by_fqn_term: dict[str, dict] = {} + self._by_version_suffix: dict[str, dict] = {} + self._tests_by_parent: dict[str, set[str]] = {} + for full_name, info in peers.items(): + self._add(full_name, info) + self._index_test_parents(full_name, info, peers) + + def _index_test_parents(self, full_name: str, info: dict, peers: dict) -> None: + """Indexes a test under each enabled testable parent.""" + if info.get("resource_type") not in {"test", "unit_test"}: + return + for parent in info.get("depends_on", {}).get("nodes", []): + if parent.startswith(_DBT_TEST_TARGET_PREFIXES) and parent in peers: + self._tests_by_parent.setdefault(parent, set()).add(full_name) + + def _add(self, full_name: str, info: dict) -> None: + """Files one node under every key a selector term could reach it by.""" + self._by_package.setdefault(info.get("package_name") or "", {})[full_name] = info + # A path containing a backslash has both POSIX and Windows interpretations. Index every possible + # base name and stem so narrowing cannot hide a collision under either runtime path flavour. + for base in _candidate_file_names(info.get("original_file_path") or ""): + for key in dict.fromkeys((base, base.rsplit(".", 1)[0] if "." in base else base)): + self._by_file.setdefault(key, {})[full_name] = info + test_name = (info.get("test_metadata") or {}).get("name") or "" + if test_name: + self._by_test_name.setdefault(test_name, {})[full_name] = info + for term in self._fqn_terms(info): + self._by_fqn_term.setdefault(term, {})[full_name] = info + if suffix := self._version_suffix(info): + self._by_version_suffix.setdefault(suffix, {})[full_name] = info + + def tests_attached_to_any(self, parent_ids: set[str]) -> set[str]: + """Returns indexed tests having at least one parent in `parent_ids`.""" + tests: set[str] = set() + for parent_id in parent_ids: + tests.update(self._tests_by_parent.get(parent_id, ())) + return tests + + @staticmethod + def _fqn_terms(info: dict) -> set[str]: + """ + Every finite fqn term that directly identifies this node under dbt's matching rules. + + The positional matcher accepts every prefix of the flattened fqn, both with and without the + package. The leaf shortcut additionally accepts the raw leaf (or the model name for a versioned + model), which can differ from a flattened prefix when it contains a dot. Version suffix matching + is indexed separately because dbt ignores any earlier selector components in that shortcut. + """ + fqn = info.get("fqn") or [] + if not fqn: + # A truncated hand-written manifest can omit fqn; the matcher then falls back to the name. + name = info.get("name") or "" + return {name} if name else set() + terms: set[str] = set() + for candidate in (fqn, fqn[1:]): + flat = _flatten_fqn(candidate) + terms.update(".".join(flat[:length]) for length in range(1, len(flat) + 1)) + if info.get("resource_type") == "model" and info.get("version") is not None and len(fqn) >= 2: + terms.add(fqn[-2]) + else: + terms.add(fqn[-1]) + return terms + + @staticmethod + def _version_suffix(info: dict) -> str: + """The suffix used by dbt's versioned-model leaf shortcut, if this node has one.""" + fqn = info.get("fqn") or [] + if info.get("resource_type") == "model" and info.get("version") is not None and len(fqn) >= 2: + return "_".join(fqn[-2:]) + return "" + + def narrow(self, terms: list[str]) -> dict: + """ + The candidate set for `terms` — a superset of the true matches, so the caller still decides. + + Returns the smallest single bucket rather than intersecting across terms: the intersection is + also sound (terms are ANDed) but measured slower, because building a set from a bucket the size + of the manifest costs more than the extra predicate evaluations it saves. + """ + smallest: dict | None = None + for term in terms: + method, _, value = term.partition(":") + if not _ or method == "fqn": + bucket = self._fqn_candidates(value if method == "fqn" else term) + elif method == "package": + bucket = self._by_package.get(value, {}) + elif method == "file": + bucket = self._by_file.get(value, {}) + elif method == "test_name": + bucket = self._by_test_name.get(value, {}) + else: + continue + if smallest is None or len(bucket) < len(smallest): + smallest = bucket + return self if smallest is None else smallest + + def _fqn_candidates(self, term: str) -> dict: + """ + The nodes an fqn `term` could match through a positional, leaf, or version-suffix rule. + + Prefixes are indexed as complete terms rather than by their first segment, keeping selectors + efficient when an ambiguous path makes `file:` unavailable. The version shortcut compares only + the selector's last two components, so its separate bucket is unioned here. + """ + candidates = dict(self._by_fqn_term.get(term, {})) + suffix = "_".join(term.split(".")[-2:]) + candidates.update(self._by_version_suffix.get(suffix, {})) + return candidates diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/dbt_task.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/dbt_task.py new file mode 100644 index 00000000..36c2d70e --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/dbt_task.py @@ -0,0 +1,173 @@ +import json +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +from databricks_dbt_factory.utils import DYNAMIC_VALUE_REFERENCE + + +_MAX_NOTEBOOK_BASE_PARAMETERS_BYTES = 1_000_000 + + +class TaskType(Enum): + """Supported task types for generated Databricks tasks.""" + + DBT = "dbt" + NOTEBOOK = "notebook" + + +@dataclass(frozen=True) +class DbtTaskOptions: + environment_key: str = "Default" # serverless env + """The key of an environment. It has to be unique within a job.""" + + catalog: str | None = None + """Optional name of the catalog to use. The value is the top level in the 3-level namespace of + Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a + warehouse_id is specified. Requires dbt-databricks >= 1.1.1.""" + + profiles_directory: str | None = None + """Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is + specified. If no warehouse_id is specified and this folder is unset, the root directory is used.""" + + project_directory: str | None = None + """Path to the project directory. Optional for Git sourced tasks, in which case if no value is + provided, the root of the Git repository is used.""" + + schema: str | None = None + """Optional schema to write to. This parameter is only used when a warehouse_id is also provided. + If not provided, the `default` schema is used.""" + + source: str | None = None + """Optional location type of the project directory. When set to `WORKSPACE`, the project will be + retrieved from the local Databricks workspace. When set to `GIT`, the project will be retrieved + from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if + `git_source` is defined and `WORKSPACE` otherwise. + + * `WORKSPACE`: Project is located in Databricks workspace. * `GIT`: Project is located in cloud + Git provider.""" + + warehouse_id: str | None = None + """ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the + profile and connection details to dbt. It can be overridden on a per-command basis by using the + `--profiles-dir` command line argument.""" + + dbt_deps_enabled: bool = False + """Optional flag to enable dbt deps to be run before each task. Defaults to False.""" + + dbt_tasks_deps: list[str] = field(default_factory=list) + """Optional comma separated list of tasks that requires dbt debs. Only in effect if dbt_deps_enabled is enabled.""" + + task_type: TaskType = TaskType.NOTEBOOK + """Task type to generate: `TaskType.NOTEBOOK` for a notebook_task wrapper (default), + `TaskType.DBT` for a native dbt_task. Strings are accepted and coerced — any value outside the + enum raises `ValueError`. Notebook mode enables base environment support and environment + variables on serverless.""" + + notebook_path: str | None = None + """Path to the dbt runner notebook. Required when task_type is `TaskType.NOTEBOOK`.""" + + job_cluster_key: str | None = None + """Job cluster key for running tasks on job compute instead of serverless.""" + + def __post_init__(self): + if not isinstance(self.task_type, TaskType): + object.__setattr__(self, "task_type", TaskType(self.task_type)) + if self.task_type is TaskType.NOTEBOOK: + if not self.notebook_path: + raise ValueError("notebook_path is required when task_type=NOTEBOOK.") + unsupported = [] + for name, value in ( + ("warehouse_id", self.warehouse_id), + ("schema", self.schema), + ("catalog", self.catalog), + ): + if value: + unsupported.append(name) + if unsupported: + raise ValueError( + f"{', '.join(unsupported)} cannot be set with task_type=NOTEBOOK; " + "notebook tasks connect via profiles.yml." + ) + + +@dataclass(frozen=True) +class DbtTask: + """Represents a dbt task in the Databricks job definition.""" + + task_key: str + commands: list[str] + options: DbtTaskOptions + depends_on: list[str] | None = None + + def to_dict(self) -> dict: + """Converts the Task to a dictionary suitable for the job definition.""" + if self.options.task_type is TaskType.NOTEBOOK: + return self._to_notebook_dict() + return self._to_dbt_dict() + + def _base_spec(self) -> dict[str, Any]: + spec: dict[str, Any] = { + "task_key": self.task_key, + "depends_on": [{"task_key": dep} for dep in (self.depends_on or [])], + } + if self.options.job_cluster_key: + spec["job_cluster_key"] = self.options.job_cluster_key + else: + spec["environment_key"] = self.options.environment_key + return spec + + def _to_dbt_dict(self) -> dict[str, Any]: + spec = self._base_spec() + dbt_task: dict[str, Any] = {"commands": self.commands} + + if self.options.source: + dbt_task["source"] = self.options.source + if self.options.project_directory: + dbt_task["project_directory"] = self.options.project_directory + if self.options.schema: + dbt_task["schema"] = self.options.schema + if self.options.warehouse_id: + dbt_task["warehouse_id"] = self.options.warehouse_id + if self.options.catalog: + dbt_task["catalog"] = self.options.catalog + if self.options.profiles_directory: + dbt_task["profiles_directory"] = self.options.profiles_directory + + spec["dbt_task"] = dbt_task + return spec + + def _to_notebook_dict(self) -> dict[str, Any]: + serialized_commands = json.dumps(self.commands) + if DYNAMIC_VALUE_REFERENCE.search(serialized_commands): + raise ValueError( + f"Notebook task {self.task_key!r} serialized dbt_commands forms a Databricks dynamic value " + f"reference; rename the selected resources or change the dbt command options." + ) + base_parameters: dict[str, str] = { + "dbt_commands": serialized_commands, + } + if self.options.project_directory: + base_parameters["project_directory"] = self.options.project_directory + if self.options.profiles_directory: + base_parameters["profiles_directory"] = self.options.profiles_directory + + serialized_size = len(json.dumps(base_parameters, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + if serialized_size > _MAX_NOTEBOOK_BASE_PARAMETERS_BYTES: + raise ValueError( + f"Notebook task {self.task_key!r} serializes base_parameters to {serialized_size:,} bytes; " + f"Databricks allows at most {_MAX_NOTEBOOK_BASE_PARAMETERS_BYTES:,} bytes. Shorten its dbt " + f"commands or path options, reduce the tests in this bundle, or generate native dbt tasks " + f"with --task-type dbt." + ) + + notebook_task: dict[str, Any] = { + "notebook_path": self.options.notebook_path, + "base_parameters": base_parameters, + } + if self.options.source: + notebook_task["source"] = self.options.source + + spec = self._base_spec() + spec["notebook_task"] = notebook_task + return spec diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/notebook/run_dbt_command.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/notebook/run_dbt_command.py new file mode 100644 index 00000000..accb8f6a --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/notebook/run_dbt_command.py @@ -0,0 +1,114 @@ +# Databricks notebook source + +import json +import os +import shlex +import shutil +import tempfile +from urllib.parse import urlparse + +from databricks.sdk import WorkspaceClient +from dbt.cli.main import dbtRunner + +# COMMAND ---------- + +dbutils.widgets.text("dbt_commands", "") +dbutils.widgets.text("project_directory", "") +dbutils.widgets.text("profiles_directory", "") + +dbt_commands = dbutils.widgets.get("dbt_commands") +project_directory = dbutils.widgets.get("project_directory") +profiles_directory = dbutils.widgets.get("profiles_directory") + +if not dbt_commands: + raise ValueError("dbt_commands parameter is required") + +# COMMAND ---------- + +# Authenticate dbt with the notebook's own credentials via the Databricks SDK. The captured token +# is short-lived (about an hour) and does not self-refresh, so a single task running longer than its +# lifetime can fail mid-run. +ws = WorkspaceClient() +os.environ["DBT_ACCESS_TOKEN"] = ws.config.authenticate()["Authorization"].removeprefix("Bearer ").strip() + +# dbt's host must be a bare hostname, but `config.host` is a full URL (scheme + host, and potentially +# a trailing slash or path). Parse it and keep only the netloc so a value like +# "https://my-workspace.databricks.com/" yields "my-workspace.databricks.com". +_parsed = urlparse(ws.config.host) +os.environ["DBT_HOST"] = _parsed.netloc or _parsed.path.strip("/") + +# chdir to the dbt project so dbt runs from inside it. Relative `project_directory` is +# resolved against this notebook's own workspace location — the same anchor native +# `dbt_task` uses. Auto-copy mode sends `.` (resolves to the notebook's dir, which is +# project root by construction); user-pinned `--notebook-path` with relative +# `--project-directory` resolves against wherever the user placed the notebook; absolute +# `project_directory` is used as-is. +if project_directory: + ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext() + notebook_dir = os.path.dirname("/Workspace" + ctx.notebookPath().get()) + target_dir = ( + project_directory + if os.path.isabs(project_directory) + else os.path.normpath(os.path.join(notebook_dir, project_directory)) + ) + os.chdir(target_dir) + +# Point dbt's artifact and log output at a private per-task dir so the parallel tasks of a job +# don't contend on the shared workspace `target/`/`logs/`. +local_dir = tempfile.mkdtemp(prefix="dbt_local_") +os.environ["DBT_TARGET_PATH"] = local_dir +os.environ["DBT_LOG_PATH"] = local_dir + + +# If a pre-built msgpack sits next to the project, deserialize it into a manifest and inject it into +# dbtRunner to skip dbt's parse phase (re-reading/hashing every file + DAG rebuild) on each task. +# Falls back to a normal parse if the msgpack is absent or unusable. +manifest = None +prebuilt_manifest_path = os.path.join("target", "partial_parse.msgpack") +if os.path.exists(prebuilt_manifest_path): + try: + from dbt.contracts.graph.manifest import Manifest + + with open(prebuilt_manifest_path, "rb") as f: + manifest = Manifest.from_msgpack(f.read()) + manifest.build_flat_graph() + print(f"[dbt-factory] injecting pre-built manifest from {prebuilt_manifest_path} (skipping dbt parse)") + except Exception as e: + print(f"[dbt-factory] manifest injection unavailable, falling back to dbt parse: {e}") + manifest = None + +try: + runner = dbtRunner(manifest=manifest) + + for command_str in json.loads(dbt_commands): + command_str = command_str.strip() + if not command_str: + continue + + if command_str.startswith("dbt "): + command_str = command_str[4:] + + args = shlex.split(command_str) + + if profiles_directory: + args.extend(["--profiles-dir", profiles_directory]) + + print(f"Running: dbt {' '.join(args)}") + print("-" * 60) + + result = runner.invoke(args) + + if not result.success: + detail = result.exception or result.result or "(no further details)" + raise RuntimeError(f"dbt command failed: dbt {' '.join(args)}\n{detail}") + + print(f"Completed successfully: dbt {' '.join(args)}") +finally: + os.environ.pop("DBT_ACCESS_TOKEN", None) + os.environ.pop("DBT_HOST", None) + os.environ.pop("DBT_TARGET_PATH", None) + os.environ.pop("DBT_LOG_PATH", None) + # Remove the private per-task target/log dir; on reused (all-purpose) clusters these + # would otherwise accumulate under the system temp dir for the life of the cluster. + if local_dir: + shutil.rmtree(local_dir, ignore_errors=True) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/task_factory.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/task_factory.py new file mode 100644 index 00000000..fe3d9b2b --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/task_factory.py @@ -0,0 +1,430 @@ +import shlex +from abc import ABC, abstractmethod +from collections.abc import Sequence + +from databricks_dbt_factory.dbt_task import DbtTask, DbtTaskOptions +from databricks_dbt_factory.utils import DYNAMIC_VALUE_REFERENCE + + +_RESERVED_DBT_SELECTION_OPTIONS = frozenset( + { + "--select", + "--models", + "--model", + "--exclude", + "--selector", + "--resource-type", + "--resource-types", + "--exclude-resource-type", + "--exclude-resource-types", + } +) +_RESERVED_DBT_PARSE_CONTEXT_OPTIONS = frozenset({"--vars", "--profile", "--profiles-dir", "--project-dir"}) +_RESERVED_DBT_SHORT_SELECTION_OPTIONS = frozenset({"-s", "-m"}) +_RESERVED_DBT_SHORT_TARGET_OPTIONS = frozenset({"-t"}) +_DBT_SHORT_OPTIONS_WITH_ATTACHED_VALUES = frozenset({"-r", "-t"}) +_DBT_TARGET_OPTIONS_WITH_SEPARATE_VALUES = frozenset({"--target", "-t"}) +_DBT_TARGET_VALUE_ERROR = "dbt target requires a nonempty value." +_EXTRA_DBT_PARSE_CONTEXT_REMEDIES = { + "--target": "--target", + "-t": "--target", + "--profiles-dir": "--profiles-directory", + "--project-dir": "--project-directory", +} + + +def _reserved_short_option(token: str, reserved_options: frozenset[str]) -> str | None: + """Returns a reserved option embedded in one dbt short-option token.""" + if not token.startswith("-") or token.startswith("--"): + return None + + for character in token[1:]: + option = f"-{character}" + if option in reserved_options: + return option + if option in _DBT_SHORT_OPTIONS_WITH_ATTACHED_VALUES: + break + return None + + +def _reserved_parse_context_option(token: str) -> str | None: + """Returns a parse-context option that is unsafe at every validation boundary.""" + long_option = token.partition("=")[0] + if long_option in _RESERVED_DBT_PARSE_CONTEXT_OPTIONS: + return long_option + return None + + +def _target_option(token: str) -> str | None: + """Returns the target option represented by one token, including its short form.""" + long_option = token.partition("=")[0] + if long_option == "--target": + return long_option + return _reserved_short_option(token, _RESERVED_DBT_SHORT_TARGET_OPTIONS) + + +def _reserved_selection_option(token: str) -> str | None: + """Returns a selection option that conflicts with factory-owned selection.""" + if token == "--": + return token + long_option = token.partition("=")[0] + if long_option in _RESERVED_DBT_SELECTION_OPTIONS: + return long_option + return _reserved_short_option(token, _RESERVED_DBT_SHORT_SELECTION_OPTIONS) + + +def _is_supported_leading_target(token: str) -> bool: + """Returns whether a token is one of the controlled leading target forms.""" + return ( + token in _DBT_TARGET_OPTIONS_WITH_SEPARATE_VALUES + or token.startswith("--target=") + or (token.startswith("-t") and not token.startswith("--") and len(token) > 2) + ) + + +def _raise_parse_context_error(option: str, reject_target: bool) -> None: + """Raises the boundary-specific error for a runtime parse-context override.""" + if reject_target: + dedicated_option = _EXTRA_DBT_PARSE_CONTEXT_REMEDIES.get(option) + remedy = f" Use the dedicated {dedicated_option} argument instead." if dedicated_option else "" + raise ValueError( + f"--extra-dbt-command-options cannot include parse-context option {option!r}.{remedy} " + "The runtime parse context must match the supplied manifest." + ) + raise ValueError( + f"dbt command options cannot include parse-context option {option!r}; " + "the runtime parse context must match the supplied manifest." + ) + + +def _validate_dbt_option_token(token: str, token_index: int, reject_target: bool) -> bool: + """Validates one option token and reports whether its next token is a target value.""" + reserved_option = _reserved_selection_option(token) + if reserved_option is not None: + raise ValueError( + f"dbt command options cannot include selection option {reserved_option!r}; " + f"the factory owns selection so each task addresses only its intended resource." + ) + + target_option = _target_option(token) + if target_option is not None: + if reject_target: + _raise_parse_context_error(target_option, reject_target=True) + if token_index != 0 or not _is_supported_leading_target(token): + raise ValueError( + "dbt command options can include at most one target, only as the leading " + f"factory-controlled option; found {target_option!r} elsewhere." + ) + if token == "--target=": + raise ValueError(_DBT_TARGET_VALUE_ERROR) + return token in _DBT_TARGET_OPTIONS_WITH_SEPARATE_VALUES + + parse_context_option = _reserved_parse_context_option(token) + if parse_context_option is not None: + _raise_parse_context_error(parse_context_option, reject_target) + return False + + +def _validate_dbt_options(dbt_options: str, reject_target: bool) -> None: + """Rejects options that can invalidate the factory's manifest-derived task guarantees.""" + if DYNAMIC_VALUE_REFERENCE.search(dbt_options): + raise ValueError( + "dbt command options cannot contain a Databricks dynamic value reference; " + "its runtime value could change the resource selection owned by the factory." + ) + try: + tokens = shlex.split(dbt_options) + except ValueError as error: + raise ValueError(f"Cannot parse dbt command options: {error}.") from error + + option_value_expected = False + for token_index, token in enumerate(tokens): + if option_value_expected: + if not token: + raise ValueError(_DBT_TARGET_VALUE_ERROR) + option_value_expected = False + continue + option_value_expected = _validate_dbt_option_token(token, token_index, reject_target) + if option_value_expected: + raise ValueError(_DBT_TARGET_VALUE_ERROR) + + +def validate_dbt_options(dbt_options: str) -> None: + """Validates options supplied to a task factory, where its leading target remains allowed.""" + _validate_dbt_options(dbt_options, reject_target=False) + + +def validate_extra_dbt_options(dbt_options: str) -> None: + """Validates CLI extra options, including parse context managed by dedicated arguments.""" + _validate_dbt_options(dbt_options, reject_target=True) + + +class DbtDependencyResolver: + @staticmethod + def resolve(node_info: dict, task_keys: dict[str, str]) -> list[str]: + """ + Resolves every scheduled direct dbt dependency to its Databricks task key. + + Args: + node_info (dict): The dbt manifest entry for the node. + task_keys (dict[str, str]): Effective task key per scheduled dbt dependency. + + Returns: + list[str]: Resolved upstream task keys. + """ + deps = node_info.get("depends_on", {}).get("nodes", []) + resolved_deps = [] + for node_full_name in deps: + task_key = task_keys.get(node_full_name) + if task_key is not None: + resolved_deps.append(task_key) + return resolved_deps + + +class TaskFactory(ABC): + """Abstract base class for creating tasks.""" + + def __init__(self, resolver: DbtDependencyResolver, task_options: DbtTaskOptions, dbt_options: str = ""): + """ + Initializes the TaskFactory. + + Args: + resolver (DbtDependencyResolver): An instance of DbtDependencyResolver to resolve dependencies. + task_options (DbtTaskOptions): Options for the task. + dbt_options (str, optional): Additional DBT options. Defaults to "". + """ + self.resolver = resolver + self.task_options = task_options + self.dbt_options = dbt_options + + @property + def dbt_options(self) -> str: + """The validated dbt options appended to every command emitted by this factory.""" + return self._dbt_options + + @dbt_options.setter + def dbt_options(self, value: str) -> None: + validate_dbt_options(value) + self._dbt_options = value + + @abstractmethod + def create_task( + self, select: str, deps_command_name: str, dbt_node_info: dict, task_key: str, task_keys: dict[str, str] + ) -> DbtTask: + """ + Abstract method to create a task. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used by `get_dbt_deps_command` to decide whether + to prepend `dbt deps` (matched against `--dbt-tasks-deps`). + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + + Returns: + DbtTask: An instance of Task. + """ + + def get_dbt_deps_command(self, dbt_task_name: str) -> str | None: + """Adds the dbt deps command if enabled and applicable. + Only return the command if enabled, and available in the task deps or not specific tasks provided. + + Args: + dbt_task_name (str): Name of the DBT task. + """ + if self.task_options.dbt_deps_enabled and ( + not self.task_options.dbt_tasks_deps or dbt_task_name in self.task_options.dbt_tasks_deps + ): + return self._build_dbt_command("deps") + return None + + def _build_dbt_command( + self, + subcommand: str, + *, + select: str | Sequence[str] | None = None, + indirect_selection: str | None = None, + ) -> str: + """ + Builds and validates one complete dbt command. + + Several selectors are passed as repeated `--select` arguments rather than joined into one, so + the union does not depend on the consumer preserving shell quoting. dbt unions repeated + occurrences; confirmed on dbt 1.12.0 under both `empty` and `cautious` indirect selection. + """ + parts = ["dbt", subcommand] + if select is not None: + selectors = [select] if isinstance(select, str) else list(select) + for selector in selectors: + parts.extend(("--select", shlex.quote(selector))) + if self.dbt_options: + parts.append(self.dbt_options) + if indirect_selection is not None: + parts.extend(("--indirect-selection", shlex.quote(indirect_selection))) + + command = " ".join(parts) + if DYNAMIC_VALUE_REFERENCE.search(command): + raise ValueError( + "The final dbt command contains a Databricks dynamic value reference; rename the " + "selected resource or change the dbt command options so no `{{...}}` reference is formed." + ) + return command + + +class ModelTaskFactory(TaskFactory): + """Factory for creating model tasks.""" + + def create_task( + self, select: str, deps_command_name: str, dbt_node_info: dict, task_key: str, task_keys: dict[str, str] + ) -> DbtTask: + """ + Creates a model task. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used to decide whether to prepend `dbt deps`. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + + Returns: + DbtTask: An instance of Task. + """ + depends_on = self.resolver.resolve(dbt_node_info, task_keys) + + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(self._build_dbt_command("run", select=select)) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class SnapshotTaskFactory(TaskFactory): + """Factory for creating snapshot tasks.""" + + def create_task( + self, select: str, deps_command_name: str, dbt_node_info: dict, task_key: str, task_keys: dict[str, str] + ) -> DbtTask: + """ + Creates a snapshot task. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used to decide whether to prepend `dbt deps`. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + + Returns: + DbtTask: An instance of Task. + """ + depends_on = self.resolver.resolve(dbt_node_info, task_keys) + + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(self._build_dbt_command("snapshot", select=select)) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class SeedTaskFactory(TaskFactory): + """Factory for creating seed tasks.""" + + def create_task( + self, select: str, deps_command_name: str, dbt_node_info: dict, task_key: str, task_keys: dict[str, str] + ) -> DbtTask: + """ + Creates a seed task. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used to decide whether to prepend `dbt deps`. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + + Returns: + DbtTask: An instance of Task. + """ + depends_on = self.resolver.resolve(dbt_node_info, task_keys) + + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(self._build_dbt_command("seed", select=select)) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class TestTaskFactory(TaskFactory): + """Factory for creating test tasks.""" + + def create_task( + self, + select: str, + deps_command_name: str, + dbt_node_info: dict, + task_key: str, + task_keys: dict[str, str], + indirect_selection: str = "empty", + ) -> DbtTask: + """ + Creates a test task for a single dbt test node. + + Args: + select (str): dbt `--select` argument identifying the node (its full dot-joined FQN). + deps_command_name (str): Bare node name used to decide whether to prepend `dbt deps`. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + task_keys (dict[str, str]): Task key per dbt node, for resolving dependencies. + indirect_selection (str): dbt indirect-selection mode required by the selection plan. + + Returns: + DbtTask: An instance of Task. + """ + depends_on = self.resolver.resolve(dbt_node_info, task_keys) + + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + # The plan's mode is appended after user options so the command cannot override the selector's + # correctness contract. Direct test selectors use `empty`; parent-scoped selectors use `cautious`. + commands.append(self._build_dbt_command("test", select=select, indirect_selection=indirect_selection)) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + def create_bundled_task( + self, + task_key: str, + selects_by_indirect_selection: dict[str, list[str]], + deps_command_name: str, + depends_on: list[str], + ) -> DbtTask: + """ + Creates one Databricks task that runs an exact set of single-resource tests. + + Test selectors with the same indirect-selection mode are joined as one dbt union. Keeping + modes separate preserves each selector plan's exactness while limiting a bundle to at most + two test commands (`empty` and `cautious`). + + Args: + task_key (str): Key for the bundled task. + selects_by_indirect_selection (dict[str, list[str]]): Exact test selectors grouped by + the indirect-selection mode required by their selection plans. + deps_command_name (str): Name used by `get_dbt_deps_command` to decide whether to prepend `dbt deps`. + depends_on (list[str]): Upstream task keys this bundled task should gate on. + + Returns: + DbtTask: An instance of Task. + """ + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + unsupported_modes = set(selects_by_indirect_selection) - {"empty", "cautious"} + if unsupported_modes: + raise ValueError(f"Unsupported dbt indirect-selection modes: {', '.join(sorted(unsupported_modes))}.") + for indirect_selection in ("empty", "cautious"): + if indirect_selection not in selects_by_indirect_selection: + continue + selects = sorted(selects_by_indirect_selection[indirect_selection]) + commands.append(self._build_dbt_command("test", select=selects, indirect_selection=indirect_selection)) + + return DbtTask(task_key, commands, self.task_options, depends_on) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/utils.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/utils.py new file mode 100644 index 00000000..02ff51a2 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/utils.py @@ -0,0 +1,255 @@ +import hashlib +import json +import re +from collections.abc import Iterable + +# Databricks substitutes complete dynamic value references in task string fields before execution. +DYNAMIC_VALUE_REFERENCE = re.compile(r"\{\{[^{}]+\}\}") + +# Databricks caps task keys at 100 characters (letters, numbers, underscores, hyphens). +MAX_TASK_KEY_LENGTH = 100 + +# dbt resource types whose task key is `_` (the type doubles as the suffix). +_SUFFIXED_TYPES = frozenset({"model", "seed", "snapshot"}) + + +def _resource_name(unique_id: str) -> str: + """ + The dbt resource name — everything after `..` — with dots turned into + underscores. Keeps a versioned model's version, e.g. `model.shop.dim.v2` -> `dim_v2`. + """ + return "_".join(unique_id.split(".")[2:]) + + +def _source_key(parts: list[str], with_package: bool) -> str: + """ + Test-task key for a source (`source...
`): `_
_test`, + package-prefixed (`__
_test`) when disambiguating a collision. + """ + prefix = f"{parts[1]}_" if with_package else "" + return f"{prefix}{parts[2]}_{parts[3]}_test" + + +def _sanitized_id(unique_id: str) -> str: + """Fallback key for an unhandled resource type: the sanitized `unique_id` (dots -> underscores).""" + return unique_id.replace(".", "_") + + +# Minimum dot-separated segment count required to build a key for each resource type. dbt ids are +# `..[...]`; sources are `source...
`. +_MIN_SEGMENTS = {"model": 3, "seed": 3, "snapshot": 3, "test": 3, "source": 4} + + +def _split_unique_id(unique_id: str) -> list[str]: + """ + Splits a dbt node `unique_id` into its dot-separated parts, validating it has enough segments + for its resource type. Raises `ValueError` with a clear message on a malformed id (e.g. a + name-less `model.pkg`, or a source missing its table). + """ + parts = unique_id.split(".") + minimum = _MIN_SEGMENTS.get(parts[0]) + if minimum is not None and len(parts) < minimum: + raise ValueError(f"Malformed dbt node id {unique_id!r}: expected at least {minimum} dot-separated segments.") + return parts + + +def generate_task_key(unique_id: str) -> str: + """ + Builds a readable Databricks task key from a dbt node `unique_id`. + + The key is based on the dbt resource *name* (not the fully-qualified id), so the package + prefix and dbt's test-name hash never appear, with the resource type as a suffix: + + * `model.shop.orders` -> `orders_model` + * `seed.shop.countries` -> `countries_seed` + * `snapshot.shop.orders_snap` -> `orders_snap_snapshot` + * `test.shop.unique_orders_id.9a1` -> `unique_orders_id_test` (hash dropped) + * `source.shop.raw.customers` -> `raw_customers_test` + + Distinct nodes can map to the same plain key (the same custom test name on two models, a + model name reused across packages), so job generation resolves keys through + `build_task_key_maps`, which keeps these plain keys and disambiguates only actual + collisions. Over-long test keys are truncated and disambiguated with dbt's hash to stay + within the task-key length limit. + """ + parts = _split_unique_id(unique_id) + resource_type = parts[0] + + if resource_type in _SUFFIXED_TYPES: + return f"{_resource_name(unique_id)}_{resource_type}" + + if resource_type == "source": + # A source only ever surfaces as a test task: source...
. + return _source_key(parts, with_package=False) + + if resource_type == "test": + # test..[.] -> _test (drop dbt's uniqueness hash). + test_hash = parts[3] if len(parts) > 3 else "" + return _bounded_test_key(parts[2], test_hash) + + # Unknown type: fall back to the sanitized id (still unique). + return _sanitized_id(unique_id) + + +def bundled_test_key(unique_id: str) -> str: + """ + Key for the single `dbt test` task that gates a tested resource in bundled mode: + `model.shop.orders` -> `orders_test`; `source.shop.raw.customers` -> `raw_customers_test`. + """ + parts = _split_unique_id(unique_id) + if parts[0] == "source": + return _source_key(parts, with_package=False) + return f"{_resource_name(unique_id)}_test" + + +def build_task_key_maps( + task_ids: Iterable[str], bundled_test_ids: Iterable[str] = () +) -> tuple[dict[str, str], dict[str, str]]: + """ + Assigns every task-producing dbt node a unique Databricks task key. + + `task_ids` are nodes that become their own task (keys from `generate_task_key`); + `bundled_test_ids` are tested resources that additionally get a bundled test task in + bundled mode (keys from `bundled_test_key`). Plain keys are kept untouched unless + several nodes claim the same key; each claimant of a contested key then gets a + disambiguated key with dbt's test hash (or the package name, when there is no hash) folded + in, falling back to the sanitized `unique_id`. Every assigned key is passed through + `_reserve`, which enforces both uniqueness and the `MAX_TASK_KEY_LENGTH` limit, so the + returned keys are always unique and within length — a valid dbt project cannot fail + deployment with a duplicate or over-long task key. + + Returns `(task_keys, bundled_test_keys)`, both keyed by `unique_id`. + """ + claims: dict[str, list[tuple[str, bool]]] = {} + for uid in task_ids: + claims.setdefault(generate_task_key(uid), []).append((uid, False)) + for uid in bundled_test_ids: + claims.setdefault(bundled_test_key(uid), []).append((uid, True)) + + task_keys: dict[str, str] = {} + bundled_test_keys: dict[str, str] = {} + taken: set[str] = set() + # Both passes walk their claims in sorted order rather than manifest order. `_reserve`'s + # numeric suffix is handed out first-come, so iterating in manifest order would let the + # *arrangement* of the manifest decide which of two colliding nodes keeps the readable key — + # adding or renaming an unrelated model could then silently swap two task keys, repointing + # per-task run history and task-level alerts at a different model. Sorting makes every + # assigned key a function of the node ids alone. + ordered_claims = sorted((key, sorted(claimants)) for key, claimants in claims.items()) + # Assign uncontested keys first so a contested claimant's fallback never steals a plain key. + for key, claimants in ordered_claims: + if len(claimants) == 1: + uid, is_bundled = claimants[0] + (bundled_test_keys if is_bundled else task_keys)[uid] = _reserve(key, taken) + for key, claimants in ordered_claims: + if len(claimants) == 1: + continue + for uid, is_bundled in claimants: + candidate = _disambiguated_bundled_test_key(uid) if is_bundled else _disambiguated_task_key(uid) + (bundled_test_keys if is_bundled else task_keys)[uid] = _reserve(candidate, taken) + return task_keys, bundled_test_keys + + +def _reserve(candidate: str, taken: set[str]) -> str: + """ + Returns a unique task key at most `MAX_TASK_KEY_LENGTH` characters long, records it in + `taken`, and never returns the same key twice. + + `candidate` is first bounded to the length limit (its tail replaced by a short hash of the + full candidate so distinct over-long keys stay distinct). If the bounded key is already + taken, a numeric suffix is appended and the result re-bounded, until an unused key is found. + """ + key = _bounded(candidate) + if key not in taken: + taken.add(key) + return key + counter = 2 + while True: + key = _bounded(f"{candidate}_{counter}") + if key not in taken: + taken.add(key) + return key + counter += 1 + + +def _bounded(key: str) -> str: + """ + Returns `key` unchanged if within `MAX_TASK_KEY_LENGTH`, otherwise truncates it and appends + a short hash of the full key so distinct over-long keys map to distinct bounded keys. + """ + if len(key) <= MAX_TASK_KEY_LENGTH: + return key + digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:8] + return f"{key[: MAX_TASK_KEY_LENGTH - len(digest) - 1]}_{digest}" + + +def _disambiguated_task_key(unique_id: str) -> str: + """ + Key for a node whose plain key collides with another node's: dbt's test hash (or the + package name, when there is no hash) is folded in to keep the key unique yet readable, + e.g. `test.shop.dup_check.9a1` -> `dup_check_9a1_test`, `model.pkg.orders` -> + `pkg_orders_model`. + """ + parts = unique_id.split(".") + resource_type = parts[0] + if resource_type in _SUFFIXED_TYPES: + return f"{parts[1]}_{_resource_name(unique_id)}_{resource_type}" + if resource_type == "source": + return _source_key(parts, with_package=True) + if resource_type == "test": + if len(parts) > 3: + return _hashed_test_key(parts[2], parts[3]) + return f"{parts[1]}_{parts[2]}_test" + return _sanitized_id(unique_id) + + +def _disambiguated_bundled_test_key(unique_id: str) -> str: + """Package-prefixed variant of `bundled_test_key`, for contested bundled test keys.""" + parts = unique_id.split(".") + if parts[0] == "source": + return _source_key(parts, with_package=True) + return f"{parts[1]}_{_resource_name(unique_id)}_test" + + +def _bounded_test_key(test_name: str, test_hash: str) -> str: + """`_test`, truncated and hash-disambiguated if it exceeds the key length limit.""" + key = f"{test_name}_test" + if len(key) <= MAX_TASK_KEY_LENGTH: + return key + return _hashed_test_key(test_name, test_hash) + + +def _hashed_test_key(test_name: str, test_hash: str) -> str: + """`__test`, truncating the name so the hash survives the length limit.""" + tail = (f"_{test_hash}" if test_hash else "") + "_test" + if len(test_name) + len(tail) <= MAX_TASK_KEY_LENGTH: + return test_name + tail + return test_name[: MAX_TASK_KEY_LENGTH - len(tail)] + tail + + +def read_dbt_manifest(path: str) -> dict: + """ + Reads a dbt manifest JSON file and returns its parsed content. + + Args: + path (str): Path to the manifest file. + + Returns: + dict: Parsed manifest content. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the file is not valid JSON, or is valid JSON that is not an object. A non-object + (list, scalar, null) would parse here and only fail later as an AttributeError on + `manifest.get(...)`, which `main` does not catch — a traceback instead of a clean `error:`. + """ + try: + with open(path, "r", encoding="utf-8") as file: + manifest = json.load(file) + except FileNotFoundError as e: + raise FileNotFoundError(f"Manifest file not found: {path}. Details: {e}") from e + except json.JSONDecodeError as e: + raise ValueError(f"Error parsing JSON from manifest file: {path}. Details: {e}") from e + if not isinstance(manifest, dict): + raise ValueError(f"Manifest file {path} must contain a JSON object, got {type(manifest).__name__}.") + return manifest diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/macros/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/macros/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_daily.sql b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_daily.sql new file mode 100644 index 00000000..e32736ce --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_daily.sql @@ -0,0 +1,17 @@ + +-- This model file defines a materialized view called 'orders_daily' +-- +-- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'materialized_view') }} + +select order_date, count(*) AS number_of_orders + +from {{ ref('orders_raw') }} + +-- During development, only process a smaller range of data +{% if target.name != 'prod' %} +where order_date >= '2019-08-01' and order_date < '2019-09-01' +{% endif %} + +group by order_date diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_raw.sql b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_raw.sql new file mode 100644 index 00000000..8faf8f38 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_raw.sql @@ -0,0 +1,16 @@ +-- This model file defines a streaming table called 'orders_raw' +-- +-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ +-- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'streaming_table') }} + +select + customer_name, + date(timestamp(from_unixtime(try_cast(order_datetime as bigint)))) as order_date, + order_number +from stream read_files( + "/databricks-datasets/retail-org/sales_orders/", + format => "json", + header => true +) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/schema.yml b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/schema.yml new file mode 100644 index 00000000..c17c810e --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/schema.yml @@ -0,0 +1,22 @@ + +version: 2 + +models: + - name: orders_raw + description: "Raw ingested orders" + columns: + - name: customer_name + description: "The name of a customer" + # customer_name repeats across orders, so only a not_null test applies here. + data_tests: + - not_null + + - name: orders_daily + description: "Number of orders by day" + columns: + - name: order_date + description: "The date on which orders took place" + # orders_daily groups by order_date, so it is unique and not null. + data_tests: + - unique + - not_null diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/seeds/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/seeds/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/snapshots/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/snapshots/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/tests/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/tests/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/__init__.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/conftest.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/conftest.py new file mode 100644 index 00000000..bc24236e --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/conftest.py @@ -0,0 +1,39 @@ +import pytest + +from databricks_dbt_factory.dbt_factory import DbtFactory +from databricks_dbt_factory.dbt_task import DbtTaskOptions +from databricks_dbt_factory.task_factory import ( + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, + DbtDependencyResolver, +) + + +def create_dbt_factory(bundle_tests: bool = False) -> DbtFactory: + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + environment_key="Default", + notebook_path="./notebooks/dbt_runner.py", + ) + dbt_options = "--target dev" + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + return DbtFactory(task_factories, bundle_tests=bundle_tests) + + +@pytest.fixture +def dbt_factory() -> DbtFactory: + return create_dbt_factory() + + +@pytest.fixture +def dbt_factory_bundled() -> DbtFactory: + return create_dbt_factory(bundle_tests=True) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/expected_tasks.json b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/expected_tasks.json new file mode 100644 index 00000000..c8a56f3d --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/expected_tasks.json @@ -0,0 +1,567 @@ +[ + { + "task_key": "diamonds_list_colors_model", + "depends_on": [ + { + "task_key": "diamonds_four_cs_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.diamonds_list_colors,package:dbt_demo,file:diamonds_list_colors.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "diamonds_prices_model", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.diamonds_prices,package:dbt_demo,file:diamonds_prices.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "diamonds_four_cs_model", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.diamonds_four_cs,package:dbt_demo,file:diamonds_four_cs.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "zzz_game_details_model", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.zzz_game_details,package:dbt_demo,file:zzz_game_details.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "second_dbt_model_model", + "depends_on": [ + { + "task_key": "first_dbt_model_model" + }, + { + "task_key": "not_null_first_dbt_model_id_test" + }, + { + "task_key": "unique_first_dbt_model_id_test" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model2.second_dbt_model,package:dbt_demo,file:second_dbt_model.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "first_dbt_model_model", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model2.first_dbt_model,package:dbt_demo,file:first_dbt_model.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "zzz_game_details_check_scores_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.data_tests.zzz_game_details_check_scores,package:dbt_demo,file:zzz_game_details_check_scores.sql,resource_type:test --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "zzz_game_details_check_dates_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.data_tests.zzz_game_details_check_dates,package:dbt_demo,file:zzz_game_details_check_dates.sql,resource_type:test --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "zzz_win_loss_records_check_records_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.data_tests.zzz_win_loss_records_check_records,package:dbt_demo,file:zzz_win_loss_records_check_records.sql,resource_type:test --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "diamonds_list_colors_test_test", + "depends_on": [ + { + "task_key": "diamonds_list_colors_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.data_tests.diamonds_list_colors_test,package:dbt_demo,file:diamonds_list_colors_test.sql,resource_type:test --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "country_codes_seed", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt seed --select fqn:dbt_demo.country_codes,package:dbt_demo,file:country_codes.csv,resource_type:seed --target dev\"]" + } + } + }, + { + "task_key": "unique_zzz_game_details_game_id_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.unique_zzz_game_details_game_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:unique --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_game_id_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_game_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_home_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_home,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:accepted_values --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_visitor_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_visitor,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:accepted_values --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_home_score_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_home_score,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_visitor_score_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_visitor_score,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_winner_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_winner,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:accepted_values --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_date_test", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_game_details_date,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "unique_first_dbt_model_id_test", + "depends_on": [ + { + "task_key": "first_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.unique_first_dbt_model_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:unique --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_first_dbt_model_id_test", + "depends_on": [ + { + "task_key": "first_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.not_null_first_dbt_model_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "unique_second_dbt_model_id_test", + "depends_on": [ + { + "task_key": "second_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.unique_second_dbt_model_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:unique --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_second_dbt_model_id_test", + "depends_on": [ + { + "task_key": "second_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.not_null_second_dbt_model_id,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_second_dbt_model_amount_usd_test", + "depends_on": [ + { + "task_key": "second_dbt_model_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model2.not_null_second_dbt_model_amount_usd,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "changed_country_model", + "depends_on": [ + { + "task_key": "country_snapshot_snapshot" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.changed_country,package:dbt_demo,file:changed_country.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "country_model", + "depends_on": [ + { + "task_key": "country_codes_seed" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.country,package:dbt_demo,file:country.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "country_snapshot_snapshot", + "depends_on": [ + { + "task_key": "country_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt snapshot --select fqn:dbt_demo.country_snapshot.country_snapshot,package:dbt_demo,file:country_snapshot.sql,resource_type:snapshot --target dev\"]" + } + } + }, + { + "task_key": "zzz_win_loss_records_model", + "depends_on": [ + { + "task_key": "zzz_game_details_model" + }, + { + "task_key": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_test" + }, + { + "task_key": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_test" + }, + { + "task_key": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_test" + }, + { + "task_key": "not_null_zzz_game_details_date_test" + }, + { + "task_key": "not_null_zzz_game_details_game_id_test" + }, + { + "task_key": "not_null_zzz_game_details_home_score_test" + }, + { + "task_key": "not_null_zzz_game_details_home_test" + }, + { + "task_key": "not_null_zzz_game_details_visitor_score_test" + }, + { + "task_key": "not_null_zzz_game_details_visitor_test" + }, + { + "task_key": "not_null_zzz_game_details_winner_test" + }, + { + "task_key": "unique_zzz_game_details_game_id_test" + }, + { + "task_key": "zzz_game_details_check_dates_test" + }, + { + "task_key": "zzz_game_details_check_scores_test" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select fqn:dbt_demo.sql_model1.zzz_win_loss_records,package:dbt_demo,file:zzz_win_loss_records.sql,resource_type:model --target dev\"]" + } + } + }, + { + "task_key": "unique_zzz_win_loss_records_team_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.unique_zzz_win_loss_records_team,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:unique --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_win_loss_records_team_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_win_loss_records_team,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_win_loss_records_wins_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_win_loss_records_wins,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "not_null_zzz_win_loss_records_losses_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.not_null_zzz_win_loss_records_losses,package:dbt_demo,file:schemas.yml,resource_type:test,test_name:not_null --target dev --indirect-selection empty\"]" + } + } + }, + { + "task_key": "unit_test_dbt_demo_diamonds_list_colors_test_get_distinct_colors", + "depends_on": [ + { + "task_key": "diamonds_list_colors_model" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select fqn:dbt_demo.models.sql_model1.diamonds_list_colors.test_get_distinct_colors,package:dbt_demo,file:test_get_distinct_colors.yml,resource_type:unit_test --target dev --indirect-selection empty\"]" + } + } + } +] diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/manifest.json b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/manifest.json new file mode 100644 index 00000000..44282d07 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/manifest.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": "1.8.7", "generated_at": "2024-12-06T13:55:35.349350Z", "invocation_id": "a2d9eb4c-6787-4ede-b5ce-44e61f3769b2", "env": {}, "project_name": "dbt_demo", "project_id": "f005e70093f1ddd3daa6316f5ec6f69e", "user_id": "b802ff80-7d9c-4428-8a39-5f56cf164bf8", "send_anonymous_usage_stats": true, "adapter_type": "databricks"}, "nodes": {"model.dbt_demo.diamonds_list_colors": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_list_colors", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_list_colors.sql", "original_file_path": "src/models/sql_model1/diamonds_list_colors.sql", "unique_id": "model.dbt_demo.diamonds_list_colors", "fqn": ["dbt_demo", "sql_model1", "diamonds_list_colors"], "alias": "diamonds_list_colors", "checksum": {"name": "sha256", "checksum": "fb8e076fbc40ae30c2441fbc660922d1d3428cf1557cc8f400dbd69caa78c266"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.194898, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_list_colors`", "raw_code": "-- create a view\nselect distinct color\nfrom {{ ref('diamonds_four_cs') }}\nsort by color asc", "language": "sql", "refs": [{"name": "diamonds_four_cs", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_four_cs"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "compiled": true, "compiled_code": "-- create a view\nselect distinct color\nfrom `dbt_demo`.`dbt_dev`.`diamonds_four_cs`\nsort by color asc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_prices": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_prices", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_prices.sql", "original_file_path": "src/models/sql_model1/diamonds_prices.sql", "unique_id": "model.dbt_demo.diamonds_prices", "fqn": ["dbt_demo", "sql_model1", "diamonds_prices"], "alias": "diamonds_prices", "checksum": {"name": "sha256", "checksum": "809b1318095c2871ab3e9d98af3cc11233e1fbb791e19682fbb38070074e53d2"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.207654, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_prices`", "raw_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum({{ cents_to_dollars('usage_quantity') }}) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "compiled": true, "compiled_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum(\n (usage_quantity / 100)::numeric(16, 2)\n) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_four_cs": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_four_cs", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_four_cs.sql", "original_file_path": "src/models/sql_model1/diamonds_four_cs.sql", "unique_id": "model.dbt_demo.diamonds_four_cs", "fqn": ["dbt_demo", "sql_model1", "diamonds_four_cs"], "alias": "diamonds_four_cs", "checksum": {"name": "sha256", "checksum": "85a0d2493625371fdfbd2d8261dcc22335301c88e08791638ad760e6cb7dd614"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.264364, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_four_cs`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom {{ source('raw', 'diamonds') }} -- equivalent of: from hive_metastore.default.diamonds", "language": "sql", "refs": [], "sources": [["raw", "diamonds"]], "metrics": [], "depends_on": {"macros": [], "nodes": ["source.dbt_demo.raw.diamonds"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom `hive_metastore`.`default`.`diamonds` -- equivalent of: from hive_metastore.default.diamonds", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.zzz_game_details": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_game_details", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_game_details.sql", "original_file_path": "src/models/sql_model1/zzz_game_details.sql", "unique_id": "model.dbt_demo.zzz_game_details", "fqn": ["dbt_demo", "sql_model1", "zzz_game_details"], "alias": "zzz_game_details", "checksum": {"name": "sha256", "checksum": "07a0ddce4e171f5838f27124b3574a77cbab23fac66bba7a1b8f87f6bfbd653f"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {"game_id": {"name": "game_id", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home": {"name": "home", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor": {"name": "visitor", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home_score": {"name": "home_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor_score": {"name": "visitor_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "winner": {"name": "winner", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "date": {"name": "date", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.43343, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_game_details`", "raw_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.{{ target.name }}_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "compiled": true, "compiled_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.dev_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.second_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "second_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/second_dbt_model.sql", "original_file_path": "src/models/sql_model2/second_dbt_model.sql", "unique_id": "model.dbt_demo.second_dbt_model", "fqn": ["dbt_demo", "sql_model2", "second_dbt_model"], "alias": "second_dbt_model", "checksum": {"name": "sha256", "checksum": "226c0e1d47e6f9bee7430c7acffadc0d239fa185558a2de703c5f0ab0b01d218"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}, "amount_usd": {"name": "amount_usd", "description": "The primary key for this table", "meta": {}, "data_type": "decimal", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}}, "created_at": 1733486794.536133, "relation_name": "`dbt_demo`.`dbt_dev`.`second_dbt_model`", "raw_code": "-- Use the `ref` function to select from other models\n\nselect\n *, {{ cents_to_dollars('id') }} as amount_usd\nfrom {{ ref('first_dbt_model') }}\nwhere id = 1", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "compiled": true, "compiled_code": "-- Use the `ref` function to select from other models\n\nselect\n *, \n (id / 100)::numeric(16, 2)\n as amount_usd\nfrom `dbt_demo`.`dbt_dev`.`first_dbt_model`\nwhere id = 1", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "d4b533efdb37067787094e25d1860ddcc473cb5cf92ebe2c358011ff2866658f"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.first_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "first_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/first_dbt_model.sql", "original_file_path": "src/models/sql_model2/first_dbt_model.sql", "unique_id": "model.dbt_demo.first_dbt_model", "fqn": ["dbt_demo", "sql_model2", "first_dbt_model"], "alias": "first_dbt_model", "checksum": {"name": "sha256", "checksum": "91d9a8b1e5c899463e52c50492b7b3c7df99fa6635c868f9165ca83e5f3b13f7"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}, "materialized": "table"}, "created_at": 1733486794.534828, "relation_name": "`dbt_demo`.`dbt_dev`.`first_dbt_model`", "raw_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n{{ config(materialized='table') }}\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "compiled": true, "compiled_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "a31b0e38eb3a9f0c0ed2f45bd615732c39e48b7f698f50a36748a687fd6d7289"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.zzz_game_details_check_scores": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_scores", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_scores.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_scores.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_scores", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_scores"], "alias": "zzz_game_details_check_scores", "checksum": {"name": "sha256", "checksum": "49cca9739f2738a4609833d267ade1a137756523f0659ec02f0bd31adb3c5789"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.31489, "relation_name": null, "raw_code": "-- This sport allows no negative scores or tie games.\n-- For this test to pass, this query must return no results.\n\nselect home_score, visitor_score\nfrom {{ ref('zzz_game_details') }}\nwhere home_score < 0\nor visitor_score < 0\nor home_score = visitor_score", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_game_details_check_dates": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_dates", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_dates.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_dates.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_dates", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_dates"], "alias": "zzz_game_details_check_dates", "checksum": {"name": "sha256", "checksum": "ac2ae3e40e5b734327e8a60a8c944e535c472314feee78d9320da1f7ed515918"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3213592, "relation_name": null, "raw_code": "-- This season's games happened between 2020-12-12 and 2021-02-06.\n-- For this test to pass, this query must return no results.\n\nselect date\nfrom {{ ref('zzz_game_details') }}\nwhere date < '2020-12-12'\nor date > '2021-02-06'", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_win_loss_records_check_records": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_win_loss_records_check_records", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_win_loss_records_check_records.sql", "original_file_path": "tests/data_tests/zzz_win_loss_records_check_records.sql", "unique_id": "test.dbt_demo.zzz_win_loss_records_check_records", "fqn": ["dbt_demo", "data_tests", "zzz_win_loss_records_check_records"], "alias": "zzz_win_loss_records_check_records", "checksum": {"name": "sha256", "checksum": "8840acce153171ad015abfe10626d1220cc9533e98c859a2b483f0d25d2e4903"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.323633, "relation_name": null, "raw_code": "-- Each team participated in 4 games this season.\n-- For this test to pass, this query must return no results.\n\nselect wins, losses\nfrom {{ ref('zzz_win_loss_records') }}\nwhere wins < 0 or wins > 4\nor losses < 0 or losses > 4\nor (wins + losses) > 4", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.diamonds_list_colors_test": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "diamonds_list_colors_test", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/diamonds_list_colors_test.sql", "original_file_path": "tests/data_tests/diamonds_list_colors_test.sql", "unique_id": "test.dbt_demo.diamonds_list_colors_test", "fqn": ["dbt_demo", "data_tests", "diamonds_list_colors_test"], "alias": "diamonds_list_colors_test", "checksum": {"name": "sha256", "checksum": "4a4d3ee2fa6aad9a7fc4b248111aa6432b67374e217793fbf29d92c2504f13be"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3264809, "relation_name": null, "raw_code": "SELECT color, COUNT(*)\nFROM {{ ref('diamonds_list_colors') }}\nGROUP BY color\nHAVING COUNT(*) > 1", "language": "sql", "refs": [{"name": "diamonds_list_colors", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "seed.dbt_demo.country_codes": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_codes", "resource_type": "seed", "package_name": "dbt_demo", "path": "country_codes.csv", "original_file_path": "src/seeds/country_codes.csv", "unique_id": "seed.dbt_demo.country_codes", "fqn": ["dbt_demo", "country_codes"], "alias": "country_codes", "checksum": {"name": "sha256", "checksum": "b54c38e4337b053b07cdc37fa5364e8f159b4e37b77d36281a8d2d11a2c09b8c"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "delimiter": ",", "quote_columns": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.388313, "relation_name": "`dbt_demo`.`dbt_dev`.`country_codes`", "raw_code": "", "root_path": "/Users/marcin.wojtyczka/Documents/repos/dbt_demo", "depends_on": {"macros": []}}, "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_game_details_game_id"], "alias": "unique_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.462498, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "unique", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_game_id"], "alias": "not_null_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4634, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home"], "alias": "not_null_zzz_game_details_home", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.464213, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3"}, "created_at": 1733486794.4649322, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor"], "alias": "not_null_zzz_game_details_visitor", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.474986, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272"}, "created_at": 1733486794.4757178, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home_score"], "alias": "not_null_zzz_game_details_home_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4776351, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor_score"], "alias": "not_null_zzz_game_details_visitor_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4783318, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_winner", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_winner.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_winner"], "alias": "not_null_zzz_game_details_winner", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.479018, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca"}, "created_at": 1733486794.4797802, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_date", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_date.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_date"], "alias": "not_null_zzz_game_details_date", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4815848, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "date", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "date", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_first_dbt_model_id.9dda05743e", "fqn": ["dbt_demo", "models", "sql_model2", "unique_first_dbt_model_id"], "alias": "unique_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.536581, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_first_dbt_model_id"], "alias": "not_null_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537291, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4", "fqn": ["dbt_demo", "models", "sql_model2", "unique_second_dbt_model_id"], "alias": "unique_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537977, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_id"], "alias": "not_null_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.538719, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_amount_usd", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_amount_usd.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_amount_usd"], "alias": "not_null_second_dbt_model_amount_usd", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.539395, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "amount_usd", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "amount_usd", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "model.dbt_demo.changed_country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "changed_country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/changed_country.sql", "original_file_path": "src/models/sql_model1/changed_country.sql", "unique_id": "model.dbt_demo.changed_country", "fqn": ["dbt_demo", "sql_model1", "changed_country"], "alias": "changed_country", "checksum": {"name": "sha256", "checksum": "e1f9599d0e633856772d321bfc75b89ead550df1ba8fe3dc40a8102c9cc2eb9d"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/changed_country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733490496.062186, "relation_name": "`dbt_demo`.`dbt_dev`.`changed_country`", "raw_code": "{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect * from {{ ref('country_snapshot') }}", "language": "sql", "refs": [{"name": "country_snapshot", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["snapshot.dbt_demo.country_snapshot"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/changed_country.sql", "compiled": true, "compiled_code": "\nselect * from `dbt_demo`.`dbt_dev`.`country_snapshot`", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/country.sql", "original_file_path": "src/models/sql_model1/country.sql", "unique_id": "model.dbt_demo.country", "fqn": ["dbt_demo", "sql_model1", "country"], "alias": "country", "checksum": {"name": "sha256", "checksum": "be347ee69b8c27dec884b8bd692e1aa0ffdac4fb2578c410d048dc9ed16b0b08"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733492304.094096, "relation_name": "`dbt_demo`.`dbt_dev`.`country`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n country_code,\n country_name\nfrom {{ ref('country_codes') }} -- use seed data to populate the table", "language": "sql", "refs": [{"name": "country_codes", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["seed.dbt_demo.country_codes"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/country.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n country_code,\n country_name\nfrom `dbt_demo`.`dbt_dev`.`country_codes` -- use seed data to populate the table", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "snapshot.dbt_demo.country_snapshot": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_snapshot", "resource_type": "snapshot", "package_name": "dbt_demo", "path": "country_snapshot.sql", "original_file_path": "src/snapshots/country_snapshot.sql", "unique_id": "snapshot.dbt_demo.country_snapshot", "fqn": ["dbt_demo", "country_snapshot", "country_snapshot"], "alias": "country_snapshot", "checksum": {"name": "sha256", "checksum": "100b8c3c320732e98704fe520db0c0bcbf9f1fccf565fccd5807b87cd0d01984"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "country_code", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "strategy": "check", "target_schema": "dbt_dev", "target_database": "dbt_demo", "updated_at": null, "check_cols": ["country_code"]}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"target_database": "dbt_demo", "target_schema": "dbt_dev", "unique_key": "country_code", "strategy": "check", "check_cols": ["country_code"]}, "created_at": 1733493205.419168, "relation_name": "`dbt_demo`.`dbt_dev`.`country_snapshot`", "raw_code": "\n\n{% set unique_key = 'country_code' %}\n{% set strategy = 'check' %}\n{% set check_cols = ['country_code'] %}\n{% set target_database = 'dbt_demo' %}\n{% set target_schema = 'dbt_dev' %}\n\n{{\n config(\n target_database=target_database,\n target_schema=target_schema,\n unique_key=unique_key,\n strategy=strategy,\n check_cols=check_cols\n )\n}}\n\n-- SQL query for the snapshot\nSELECT *\nFROM {{ ref('country') }}\n\n", "language": "sql", "refs": [{"name": "country", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.country"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "model.dbt_demo.zzz_win_loss_records": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_win_loss_records", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_win_loss_records.sql", "original_file_path": "src/models/sql_model1/zzz_win_loss_records.sql", "unique_id": "model.dbt_demo.zzz_win_loss_records", "fqn": ["dbt_demo", "sql_model1", "zzz_win_loss_records"], "alias": "zzz_win_loss_records", "checksum": {"name": "sha256", "checksum": "2af4334f4afcfc71dbb39dd2bb88e64cf6e49fb20ec401ef0abf345aa827eb69"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {"team": {"name": "team", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "wins": {"name": "wins", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "losses": {"name": "losses", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733493328.3391528, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_win_loss_records`", "raw_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from {{ ref('zzz_game_details') }}\n)\ngroup by winner\norder by wins desc", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "compiled": true, "compiled_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from `dbt_demo`.`dbt_dev`.`zzz_game_details`\n)\ngroup by winner\norder by wins desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_win_loss_records_team"], "alias": "unique_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3774579, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "unique", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_team"], "alias": "not_null_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.378356, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_wins", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_wins.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_wins"], "alias": "not_null_zzz_win_loss_records_wins", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3790798, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "wins", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "wins", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_losses", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_losses.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_losses"], "alias": "not_null_zzz_win_loss_records_losses", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.379801, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "losses", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "losses", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}}, "sources": {"source.dbt_demo.raw.diamonds": {"database": "hive_metastore", "schema": "default", "name": "diamonds", "resource_type": "source", "package_name": "dbt_demo", "path": "src/models/sql_model1/sources.yml", "original_file_path": "src/models/sql_model1/sources.yml", "unique_id": "source.dbt_demo.raw.diamonds", "fqn": ["dbt_demo", "sql_model1", "raw", "diamonds"], "source_name": "raw", "source_description": "", "loader": "", "identifier": "diamonds", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "`hive_metastore`.`default`.`diamonds`", "created_at": 1733486794.541405}}, "macros": {"macro.dbt_demo.cents_to_dollars": {"name": "cents_to_dollars", "resource_type": "macro", "package_name": "dbt_demo", "path": "src/macros/centrs_to_dollars.sql", "original_file_path": "src/macros/centrs_to_dollars.sql", "unique_id": "macro.dbt_demo.cents_to_dollars", "macro_sql": "{% macro cents_to_dollars(column_name, scale=2) %}\n ({{ column_name }} / 100)::numeric(16, {{ scale }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.526016, "supported_languages": null}, "macro.dbt_databricks.materialization_materialized_view_databricks": {"name": "materialization_materialized_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialization_materialized_view_databricks", "macro_sql": "{% materialization materialized_view, adapter = 'databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.materialized_view_get_build_sql", "macro.dbt_databricks.materialized_view_execute_no_op", "macro.dbt_databricks.materialized_view_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5290601, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt_databricks.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.530187, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5303931, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.531114, "supported_languages": null}, "macro.dbt_databricks.materialization_view_databricks": {"name": "materialization_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.materialization_view_databricks", "macro_sql": "{% materialization view, adapter='databricks' -%}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and not old_relation.is_view -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.532544, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_table_databricks": {"name": "materialization_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_databricks.materialization_table_databricks", "macro_sql": "{% materialization table, adapter = 'databricks', supported_languages=['sql', 'python'] %}\n {{ log(\"MATERIALIZING TABLE\") }}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation and (not (old_relation.is_delta and config.get('file_format', default='delta') == 'delta')) or (old_relation.is_materialized_view or old_relation.is_streaming_table) -%}\n {{ adapter.drop_relation(old_relation) }}\n {%- endif %}\n\n -- build model\n\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% if language==\"python\" %}\n {% do apply_tblproperties(target_relation, tblproperties) %}\n {% endif %}\n {%- do apply_tags(target_relation, tags) -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.apply_tags", "macro.dbt.persist_docs", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5349078, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks_build_snapshot_staging_table": {"name": "databricks_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro_sql": "{% macro databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=target_relation.database,\n type='view') -%}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n create or replace view {{ tmp_relation }}\n as\n {{ select }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5406778, "supported_languages": null}, "macro.dbt_databricks.materialization_snapshot_databricks": {"name": "materialization_snapshot_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.materialization_snapshot_databricks", "macro_sql": "{% materialization snapshot, adapter='databricks' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'delta') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = databricks__get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table',\n needs_information=True) -%}\n\n {%- if file_format not in ['delta', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% if target_relation.database is none %}\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% else %}\n {% set staging_table = databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% endif %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% call statement_with_staging_table('main', staging_table) %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=True) %}\n\n {% endif %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.544019, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_streaming_table_databricks": {"name": "materialization_streaming_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.materialization_streaming_table_databricks", "macro_sql": "{% materialization streaming_table, adapter='databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.StreamingTable) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% set build_sql = streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ streaming_table_execute_no_op(target_relation) }}\n {% else %}\n {{ streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.streaming_table_get_build_sql", "macro.dbt_databricks.streaming_table_execute_no_op", "macro.dbt_databricks.streaming_table_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.546883, "supported_languages": ["sql"]}, "macro.dbt_databricks.streaming_table_get_build_sql": {"name": "streaming_table_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_get_build_sql", "macro_sql": "{% macro streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_streaming_table_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_streaming_table %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_streaming_table_configuration_changes(existing_relation, config) %}\n {% if configuration_changes is none %}\n {% set build_sql = refresh_streaming_table(target_relation, sql) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_streaming_table_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro.dbt_databricks.refresh_streaming_table", "macro.dbt_databricks.get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5479271, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_no_op": {"name": "streaming_table_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_no_op", "macro_sql": "{% macro streaming_table_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548121, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_build_sql": {"name": "streaming_table_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_build_sql", "macro_sql": "{% macro streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548807, "supported_languages": null}, "macro.dbt_databricks.databricks__can_clone_table": {"name": "databricks__can_clone_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__can_clone_table", "macro_sql": "{% macro databricks__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550354, "supported_languages": null}, "macro.dbt_databricks.databricks__create_or_replace_clone": {"name": "databricks__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__create_or_replace_clone", "macro_sql": "{% macro databricks__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace\n table {{ this_relation }}\n shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550466, "supported_languages": null}, "macro.dbt_databricks.materialization_clone_databricks": {"name": "materialization_clone_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.materialization_clone_databricks", "macro_sql": "\n\n{%- materialization clone, adapter='databricks' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.552143, "supported_languages": ["sql"]}, "macro.dbt_databricks.dbt_databricks_validate_get_file_format": {"name": "dbt_databricks_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro_sql": "{% macro dbt_databricks_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.553088, "supported_languages": null}, "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy": {"name": "dbt_databricks_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'merge', 'replace_where', 'append', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_delta_only_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via warehouse\n Use the 'merge' or 'replace_where' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite', 'replace_where'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy == 'replace_where' and file_format not in ['delta'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55381, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_default_sql": {"name": "databricks__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_default_sql", "macro_sql": "{% macro databricks__get_incremental_default_sql(arg_dict) %}\n {{ return(get_incremental_merge_sql(arg_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55716, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_append_sql": {"name": "databricks__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_append_sql", "macro_sql": "{% macro databricks__get_incremental_append_sql(arg_dict) %}\n {% do return(get_insert_into_sql(arg_dict[\"temp_relation\"], arg_dict[\"target_relation\"])) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5573308, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_replace_where_sql": {"name": "databricks__get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_replace_where_sql", "macro_sql": "{% macro databricks__get_incremental_replace_where_sql(arg_dict) %}\n {% do return(get_replace_where_sql(arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557461, "supported_languages": null}, "macro.dbt_databricks.get_incremental_replace_where_sql": {"name": "get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_incremental_replace_where_sql", "macro_sql": "{% macro get_incremental_replace_where_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_replace_where_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557615, "supported_languages": null}, "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql": {"name": "databricks__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql", "macro_sql": "{% macro databricks__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}\n {{ return(get_insert_overwrite_sql(source, target)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5577788, "supported_languages": null}, "macro.dbt_databricks.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558079, "supported_languages": null}, "macro.dbt_databricks.get_replace_where_sql": {"name": "get_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_replace_where_sql", "macro_sql": "{% macro get_replace_where_sql(args_dict) -%}\n {%- set predicates = args_dict['incremental_predicates'] -%}\n {%- set target_relation = args_dict['target_relation'] -%}\n {%- set temp_relation = args_dict['temp_relation'] -%}\n\n insert into {{ target_relation }}\n {% if predicates %}\n {% if predicates is sequence and predicates is not string %}\n replace where {{ predicates | join(' and ') }}\n {% else %}\n replace where {{ predicates }}\n {% endif %}\n {% endif %}\n table {{ temp_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5584981, "supported_languages": null}, "macro.dbt_databricks.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) | map(attribute=\"quoted\") | list -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) | map(attribute=\"quoted\") | list -%}\n {{ insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.insert_into_sql_impl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558819, "supported_languages": null}, "macro.dbt_databricks.insert_into_sql_impl": {"name": "insert_into_sql_impl", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.insert_into_sql_impl", "macro_sql": "{% macro insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) %}\n {%- set common_columns = [] -%}\n {%- for dest_col in dest_columns -%}\n {%- if dest_col in source_columns -%}\n {%- do common_columns.append(dest_col) -%}\n {%- else -%}\n {%- do common_columns.append('DEFAULT') -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set dest_cols_csv = dest_columns | join(', ') -%}\n {%- set source_cols_csv = common_columns | join(', ') -%}\ninsert into table {{ target_relation }} ({{ dest_cols_csv }})\nselect {{source_cols_csv}} from {{ source_relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5592902, "supported_languages": null}, "macro.dbt_databricks.databricks__get_merge_sql": {"name": "databricks__get_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_merge_sql", "macro_sql": "{% macro databricks__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set source_columns = (adapter.get_columns_in_relation(source) | map(attribute='quoted') | list)-%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} <=> DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} <=> DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n when matched then update set {{ get_merge_update_set(update_columns, on_schema_change, source_columns) }}\n when not matched then insert {{ get_merge_insert(on_schema_change, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.incremental_validate_on_schema_change", "macro.dbt.get_merge_update_columns", "macro.dbt_databricks.get_merge_update_set", "macro.dbt_databricks.get_merge_insert"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5605829, "supported_languages": null}, "macro.dbt_databricks.get_merge_update_set": {"name": "get_merge_update_set", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_update_set", "macro_sql": "{% macro get_merge_update_set(update_columns, on_schema_change, source_columns) %}\n {%- if update_columns -%}\n {%- for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- elif on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n {%- for column in source_columns -%}\n {{ column }} = DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.560978, "supported_languages": null}, "macro.dbt_databricks.get_merge_insert": {"name": "get_merge_insert", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_insert", "macro_sql": "{% macro get_merge_insert(on_schema_change, source_columns) %}\n {%- if on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n ({{ source_columns | join(\", \") }}) VALUES (\n {%- for column in source_columns -%}\n DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %})\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.561265, "supported_languages": null}, "macro.dbt_databricks.materialization_incremental_databricks": {"name": "materialization_incremental_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.materialization_incremental_databricks", "macro_sql": "{% materialization incremental, adapter='databricks', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='delta') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'merge' -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n {%- set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool -%}\n\n {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%}\n {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set incremental_predicates = config.get('predicates', default=none) or config.get('incremental_predicates', default=none) -%}\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%}\n {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%}\n {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%}\n {%- else -%}\n {%- set temp_relation_suffix = '__dbt_tmp' -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode to STATIC for initial replace --#}\n {%- if incremental_strategy == 'insert_overwrite' and should_full_refresh() -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = STATIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {% do apply_tags(target_relation, tags) %}\n {%- if language == 'python' -%}\n {%- do apply_tblproperties(target_relation, tblproperties) %}\n {%- endif -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- elif existing_relation.is_view or existing_relation.is_materialized_view or existing_relation.is_streaming_table or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% if not existing_relation.is_view %}\n {% do persist_constraints(target_relation, model) %}\n {% endif %}\n {% do apply_tags(target_relation, tags) %}\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- else -%}\n {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n {#-- Relation must be merged --#}\n {%- set _existing_config = adapter.get_relation_config(existing_relation) -%}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%}\n {%- set temp_relation = databricks__make_temp_relation(target_relation, suffix=temp_relation_suffix, as_table=language != 'sql') -%}\n {%- call statement('create_temp_relation', language=language) -%}\n {{ create_table_as(True, temp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, temp_relation, existing_relation) -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': temp_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {%- set build_sql = strategy_sql_macro_func(strategy_arg_dict) -%}\n {%- if language == 'sql' -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', temp_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why does not either drop_relation or adapter.drop_relation work here?!\n --#}\n {%- endif -%}\n {% do apply_liquid_clustered_cols(target_relation) %}\n {% if _configuration_changes is not none %}\n {% set tags = _configuration_changes.changes.get(\"tags\", None) %}\n {% set tblproperties = _configuration_changes.changes.get(\"tblproperties\", None) %}\n {% if tags is not none %}\n {% do apply_tags(target_relation, tags.set_tags, tags.unset_tags) %}\n {%- endif -%}\n {% if tblproperties is not none %}\n {% do apply_tblproperties(target_relation, tblproperties.tblproperties) %}\n {%- endif -%}\n {%- endif -%}\n {% do persist_docs(target_relation, model, for_relation=True) %}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt.persist_docs", "macro.dbt_databricks.databricks__make_temp_relation", "macro.dbt.process_schema_changes", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt_databricks.apply_liquid_clustered_cols", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.566802, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks__get_binding_char": {"name": "databricks__get_binding_char", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__get_binding_char", "macro_sql": "{% macro databricks__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5697222, "supported_languages": null}, "macro.dbt_databricks.databricks__load_csv_rows": {"name": "databricks__load_csv_rows", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__load_csv_rows", "macro_sql": "{% macro databricks__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n {% set must_cast = model['config'].get(\"file_format\", \"delta\") == \"parquet\" %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- if must_cast -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- else -%}\n {{ get_binding_char() }}\n {%- endif -%}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True, close_cursor=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.571152, "supported_languages": null}, "macro.dbt_databricks.databricks__reset_csv_table": {"name": "databricks__reset_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__reset_csv_table", "macro_sql": "{% macro databricks__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {% if old_relation.is_delta and config.get('file_format', default='delta') == 'delta' %}\n {% set sql = create_or_replace_csv_table(model, agate_table, True) %}\n {% else %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {% else %}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table", "macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57173, "supported_languages": null}, "macro.dbt_databricks.create_or_replace_csv_table": {"name": "create_or_replace_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.create_or_replace_csv_table", "macro_sql": "{% macro create_or_replace_csv_table(model, agate_table, replace=False) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {%- set column_comment = config.persist_column_docs() and model.columns %}\n {%- set identifier = model['alias'] -%}\n {%- set relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}\n {%- set replace_clause = \"\" -%}\n {%- if replace -%}\n {%- set replace_clause = \"or replace\" -%}\n {%- endif -%}\n\n {% set sql %}\n create {{replace_clause}} table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {%- set column_comment_clause = \"\" -%}\n {%- if column_comment and col_name in model.columns.keys() -%} \n {%- set comment = model.columns[col_name]['description'] | replace(\"'\", \"\\\\'\") -%}\n {%- if comment and comment != \"\" -%}\n {%- set column_comment_clause = \"comment '\" ~ comment ~ \"'\" -%}\n {%- endif -%}\n {%- endif -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {{ column_comment_clause }}{%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57331, "supported_languages": null}, "macro.dbt_databricks.databricks__create_csv_table": {"name": "databricks__create_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__create_csv_table", "macro_sql": "{% macro databricks__create_csv_table(model, agate_table) %}\n {{ return(create_or_replace_csv_table(model, agate_table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.573471, "supported_languages": null}, "macro.dbt_databricks.materialization_seed_databricks": {"name": "materialization_seed_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.materialization_seed_databricks", "macro_sql": "{% materialization seed, adapter='databricks' %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and (old_relation.is_view or old_relation.is_materialized_view)) -%}\n {%- set exists_as_streaming_table = (old_relation is not none and old_relation.is_streaming_table) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view or a materialized view\".format(old_relation)) }}\n {% elif exists_as_streaming_table %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a streaming table\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n -- No need to persist docs, already handled in seed create\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576501, "supported_languages": ["sql"]}, "macro.dbt_databricks.databricks__generate_database_name": {"name": "databricks__generate_database_name", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt_databricks.databricks__generate_database_name", "macro_sql": "{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n {{ return(default_database) }}\n {%- else -%}\n {{ return(custom_database_name) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576849, "supported_languages": null}, "macro.dbt_databricks.databricks__get_drop_sql": {"name": "databricks__get_drop_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__get_drop_sql", "macro_sql": "{% macro databricks__get_drop_sql(relation) -%}\n {%- if relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n {%- elif relation.is_streaming_table-%}\n {{ drop_streaming_table(relation) }}\n {%- elif relation.is_view -%}\n {{ drop_view(relation) }}\n {%- else -%}\n {{ drop_table(relation) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_materialized_view", "macro.dbt_databricks.drop_streaming_table", "macro.dbt.drop_view", "macro.dbt.drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577309, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_relation": {"name": "databricks__drop_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_relation", "macro_sql": "{% macro databricks__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577478, "supported_languages": null}, "macro.dbt_databricks.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {% do return(adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57809, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_sql": {"name": "databricks__get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_sql", "macro_sql": "{% macro databricks__get_replace_sql(existing_relation, target_relation, sql) %}\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_create_backup_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_drop_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n ]) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ return([\n get_create_backup_sql(existing_relation),\n get_create_sql(target_relation, sql),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ return([\n get_drop_sql(existing_relation),\n get_create_sql(target_relation, sql)\n ]) }}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.579058, "supported_languages": null}, "macro.dbt_databricks.fetch_tags": {"name": "fetch_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags", "macro_sql": "{% macro fetch_tags(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {% call statement('list_tags', fetch_result=True) -%}\n {{ fetch_tags_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_tags').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_tags_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5802681, "supported_languages": null}, "macro.dbt_databricks.fetch_tags_sql": {"name": "fetch_tags_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags_sql", "macro_sql": "{% macro fetch_tags_sql(relation) -%}\n SELECT tag_name, tag_value\n FROM `system`.`information_schema`.`table_tags`\n WHERE catalog_name = '{{ relation.database|lower }}' \n AND schema_name = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5804422, "supported_languages": null}, "macro.dbt_databricks.apply_tags": {"name": "apply_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.apply_tags", "macro_sql": "{% macro apply_tags(relation, set_tags, unset_tags=[]) -%}\n {%- if (set_tags or unset_tags) and relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif -%}\n {%- if set_tags %}\n {%- call statement('set_tags') -%}\n {{ alter_set_tags(relation, set_tags) }}\n {%- endcall -%}\n {%- endif %}\n {%- if unset_tags %}\n {%- call statement('unset_tags') -%}\n {{ alter_unset_tags(relation, unset_tags) }}\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_set_tags", "macro.dbt_databricks.alter_unset_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.580887, "supported_languages": null}, "macro.dbt_databricks.alter_set_tags": {"name": "alter_set_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_set_tags", "macro_sql": "{% macro alter_set_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} SET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' = '{{ tags[tag] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.581147, "supported_languages": null}, "macro.dbt_databricks.alter_unset_tags": {"name": "alter_unset_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_unset_tags", "macro_sql": "{% macro alter_unset_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} UNSET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n )\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58137, "supported_languages": null}, "macro.dbt_databricks.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/location.sql", "original_file_path": "macros/relations/location.sql", "unique_id": "macro.dbt_databricks.location_clause", "macro_sql": "{% macro location_clause(relation) %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- elif (not relation.is_hive_metastore()) and file_format != 'delta' -%}\n {{ exceptions.raise_compiler_error(\n 'Incompatible configuration: `location_root` must be set when using a non-delta file format with Unity Catalog'\n ) }}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5819929, "supported_languages": null}, "macro.dbt_databricks.optimize": {"name": "optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.optimize", "macro_sql": "{% macro optimize(relation) %}\n {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.582578, "supported_languages": null}, "macro.dbt_databricks.databricks__optimize": {"name": "databricks__optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.databricks__optimize", "macro_sql": "\n\n{%- macro databricks__optimize(relation) -%}\n {%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and\n var('databricks_skip_optimize', 'false')|lower != 'true' and\n config.get('file_format', 'delta') == 'delta' -%}\n {%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) -%}\n {%- call statement('run_optimize_stmt') -%}\n {{ get_optimize_sql(relation) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_optimize_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583037, "supported_languages": null}, "macro.dbt_databricks.get_optimize_sql": {"name": "get_optimize_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.get_optimize_sql", "macro_sql": "{%- macro get_optimize_sql(relation) %}\n optimize {{ relation }}\n {%- if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}\n {%- if config.get('liquid_clustered_by', False) %}\n {{ exceptions.warn(\"Both zorder and liquid_clustered_by are set but they are incompatible. zorder will be ignored.\") }}\n {%- else %}\n {%- set zorder = config.get('zorder', none) %}\n {# TODO: predicates here? WHERE ... #}\n {%- if zorder is sequence and zorder is not string %}\n zorder by (\n {%- for col in zorder %}\n {{ col }}{% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- else %}\n zorder by ({{zorder}})\n {%- endif %}\n {%- endif %}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583699, "supported_languages": null}, "macro.dbt_databricks.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/file_format.sql", "original_file_path": "macros/relations/file_format.sql", "unique_id": "macro.dbt_databricks.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {%- set file_format = config.get('file_format', default='delta') -%}\n using {{ file_format }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583901, "supported_languages": null}, "macro.dbt_databricks.liquid_clustered_cols": {"name": "liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.liquid_clustered_cols", "macro_sql": "{% macro liquid_clustered_cols() -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n CLUSTER BY ({{ cols | join(', ') }})\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5845, "supported_languages": null}, "macro.dbt_databricks.apply_liquid_clustered_cols": {"name": "apply_liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.apply_liquid_clustered_cols", "macro_sql": "{% macro apply_liquid_clustered_cols(target_relation) -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {%- call statement('set_cluster_by_columns') -%}\n ALTER {{ target_relation.type }} {{ target_relation }} CLUSTER BY ({{ cols | join(', ') }})\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.584919, "supported_languages": null}, "macro.dbt_databricks.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() -%}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58541, "supported_languages": null}, "macro.dbt_databricks.databricks__tblproperties_clause": {"name": "databricks__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.databricks__tblproperties_clause", "macro_sql": "{% macro databricks__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5857081, "supported_languages": null}, "macro.dbt_databricks.apply_tblproperties": {"name": "apply_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.apply_tblproperties", "macro_sql": "{% macro apply_tblproperties(relation, tblproperties) -%}\n {% if tblproperties %}\n {%- call statement('apply_tblproperties') -%}\n ALTER {{ relation.type }} {{ relation }} SET TBLPROPERTIES (\n {% for tblproperty in tblproperties -%}\n '{{ tblproperty }}' = '{{ tblproperties[tblproperty] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n {%- endcall -%}\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58604, "supported_languages": null}, "macro.dbt_databricks.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.599794, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_constraints": {"name": "databricks__persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__persist_constraints", "macro_sql": "{% macro databricks__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% set has_model_contract = contract_config and contract_config.enforced %}\n {% set has_databricks_constraints = config.get('persist_constraints', False) %}\n\n {% if (has_model_contract or has_databricks_constraints) %}\n {% if config.get('file_format', 'delta') != 'delta' %}\n {# Constraints are only supported for delta tables #}\n {{ exceptions.warn(\"Constraints not supported for file format: \" ~ config.get('file_format')) }}\n {% elif relation.is_view %}\n {# Constraints are not supported for views. This point in the code should not have been reached. #}\n {{ exceptions.raise_compiler_error(\"Constraints not supported for views.\") }}\n {% elif is_incremental() %}\n {# Constraints are not applied for incremental updates. This point in the code should not have been reached #}\n {{ exceptions.raise_compiler_error(\"Constraints are not applied for incremental updates. Full refresh is required to update constraints.\") }}\n {% else %}\n {% do alter_column_set_constraints(relation, model) %}\n {% do alter_table_add_constraints(relation, model) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt_databricks.alter_column_set_constraints", "macro.dbt_databricks.alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600513, "supported_languages": null}, "macro.dbt_databricks.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600693, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_table_add_constraints": {"name": "databricks__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_table_add_constraints", "macro_sql": "{% macro databricks__alter_table_add_constraints(relation, model) %}\n {% set constraints = get_model_constraints(model) %}\n {% set statements = get_constraints_sql(relation, constraints, model) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_model_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6010072, "supported_languages": null}, "macro.dbt_databricks.get_model_constraints": {"name": "get_model_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_model_constraints", "macro_sql": "{% macro get_model_constraints(model) %}\n {% set constraints = model.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and model.get('meta', {}).get('constraints') is sequence %}\n {# Databricks constraints implementation. Constraints are in the meta property. #}\n {% set db_constraints = model.get('meta', {}).get('constraints', []) %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.601491, "supported_languages": null}, "macro.dbt_databricks.get_column_constraints": {"name": "get_column_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_column_constraints", "macro_sql": "{% macro get_column_constraints(column) %}\n {% set constraints = column.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and column.get('meta', {}).get('constraint') %}\n {# Databricks constraints implementation. Constraint is in the meta property. #}\n {% set db_constraints = [column.get('meta', {}).get('constraint')] %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints, column) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6019762, "supported_languages": null}, "macro.dbt_databricks.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602155, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_set_constraints": {"name": "databricks__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_set_constraints", "macro_sql": "{% macro databricks__alter_column_set_constraints(relation, model) %}\n {% set column_dict = model.columns %}\n {% for column_name in column_dict %}\n {% set column = column_dict[column_name] %}\n {% set constraints = get_column_constraints(column) %}\n {% set statements = get_constraints_sql(relation, constraints, model, column) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602596, "supported_languages": null}, "macro.dbt_databricks.get_constraints_sql": {"name": "get_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraints_sql", "macro_sql": "{% macro get_constraints_sql(relation, constraints, model, column={}) %}\n {% set statements = [] %}\n -- Hack so that not null constraints will be applied before primary key constraints\n {% for constraint in constraints|sort(attribute='type') %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endfor %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_constraint_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.603168, "supported_languages": null}, "macro.dbt_databricks.get_constraint_sql": {"name": "get_constraint_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraint_sql", "macro_sql": "{% macro get_constraint_sql(relation, constraint, model, column={}) %}\n {% set statements = [] %}\n {% set type = constraint.get(\"type\", \"\") %}\n\n {% if type == 'check' %}\n {% set expression = constraint.get(\"expression\", \"\") %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Invalid check constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5 (relation.identifier ~ \";\" ~ column.get(\"name\", \"\") ~ \";\" ~ expression ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" check (\" ~ expression ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'not_null' %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if column %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% set stmt = \"alter table \" ~ relation ~ \" change column \" ~ quoted_name ~ \" set not null \" ~ (constraint.expression or \"\") ~ \";\" %}\n {% do statements.append(stmt) %}\n {% else %}\n {{ exceptions.warn('not_null constraint on invalid column: ' ~ column_name) }}\n {% endif %}\n {% endfor %}\n {% elif type == 'primary_key' %}\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ type)}}\n {% endif %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid primary key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"primary_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" primary key(\" ~ joined_names ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'foreign_key' %}\n\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n \n {% if constraint.get('expression') %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ constraint.get('expression') ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key\" ~ constraint.get('expression') %}\n {% else %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid foreign key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set parent = constraint.get(\"to\") %}\n {% if not parent %}\n {{ exceptions.raise_compiler_error('No parent table defined for foreign key: ' ~ expression) }}\n {% endif %}\n {% if not \".\" in parent %}\n {% set parent = relation.schema ~ \".\" ~ parent%}\n {% endif %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\" ~ parent ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key(\" ~ joined_names ~ \") references \" ~ parent %}\n {% set parent_columns = constraint.get(\"to_columns\") %}\n {% if parent_columns %}\n {% set stmt = stmt ~ \"(\" ~ parent_columns|join(\", \") ~ \")\"%}\n {% endif %}\n {% endif %}\n {% set stmt = stmt ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif constraint.get('warn_unsupported') %}\n {{ exceptions.warn(\"unsupported constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6086242, "supported_languages": null}, "macro.dbt_databricks.databricks_constraints_to_dbt": {"name": "databricks_constraints_to_dbt", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks_constraints_to_dbt", "macro_sql": "{% macro databricks_constraints_to_dbt(constraints, column) %}\n {# convert constraints defined using the original databricks format #}\n {% set dbt_constraints = [] %}\n {% for constraint in constraints %}\n {% if constraint.get and constraint.get(\"type\") %}\n {# already in model contract format #}\n {% do dbt_constraints.append(constraint) %}\n {% else %}\n {% if column %}\n {% if constraint == \"not_null\" %}\n {% do dbt_constraints.append({\"type\": \"not_null\", \"columns\": [column.get(\"name\")]}) %}\n {% else %}\n {{ exceptions.raise_compiler_error('Invalid constraint for column ' ~ column.get(\"name\", \"\") ~ '. Only `not_null` is supported.') }}\n {% endif %}\n {% else %}\n {% set name = constraint['name'] %}\n {% if not name %}\n {{ exceptions.raise_compiler_error('Invalid check constraint name') }}\n {% endif %}\n {% set condition = constraint['condition'] %}\n {% if not condition %}\n {{ exceptions.raise_compiler_error('Invalid check constraint condition') }}\n {% endif %}\n {% do dbt_constraints.append({\"name\": name, \"type\": \"check\", \"expression\": condition}) %}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(dbt_constraints) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.609767, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_sql": {"name": "databricks__get_create_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_sql", "macro_sql": "{%- macro databricks__get_create_sql(relation, sql) -%}\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_streaming_table -%}\n {{ get_create_streaming_table_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610386, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_materialized_view": {"name": "databricks__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_materialized_view", "macro_sql": "{% macro databricks__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610508, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_materialized_view": {"name": "databricks__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_materialized_view", "macro_sql": "{% macro databricks__refresh_materialized_view(relation) -%}\n refresh materialized view {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61062, "supported_languages": null}, "macro.dbt_databricks.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611409, "supported_languages": null}, "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes": {"name": "databricks__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes", "macro_sql": "\n\n\n{%- macro databricks__get_materialized_view_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_materialized_view = adapter.get_relation_config(existing_relation) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = materialized_view.get_changeset(_existing_materialized_view) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6116831, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql": {"name": "databricks__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {% do return(get_alter_mv_internal(relation, configuration_changes)) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_mv_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611993, "supported_languages": null}, "macro.dbt_databricks.get_alter_mv_internal": {"name": "get_alter_mv_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_mv_internal", "macro_sql": "{% macro get_alter_mv_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n -- Currently only schedule can be altered\n ALTER MATERIALIZED VIEW {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, refresh.is_altered) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6122189, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql": {"name": "databricks__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_create_materialized_view_as_sql(relation, sql) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = materialized_view.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = materialized_view.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = materialized_view.config[\"comment\"].comment -%}\n {%- set refresh = materialized_view.config[\"refresh\"] -%}\n create materialized view {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.612938, "supported_languages": null}, "macro.dbt_databricks.drop_streaming_table": {"name": "drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.drop_streaming_table", "macro_sql": "{% macro drop_streaming_table(relation) -%}\n {{ return(adapter.dispatch('drop_streaming_table', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.default__drop_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.613157, "supported_languages": null}, "macro.dbt_databricks.default__drop_streaming_table": {"name": "default__drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.default__drop_streaming_table", "macro_sql": "{% macro default__drop_streaming_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6132412, "supported_languages": null}, "macro.dbt_databricks.refresh_streaming_table": {"name": "refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.refresh_streaming_table", "macro_sql": "{% macro refresh_streaming_table(relation, sql) -%}\n {{ adapter.dispatch('refresh_streaming_table', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61346, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_streaming_table": {"name": "databricks__refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_streaming_table", "macro_sql": "{% macro databricks__refresh_streaming_table(relation, sql) -%}\n create or refresh streaming table {{ relation }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6135712, "supported_languages": null}, "macro.dbt_databricks.get_alter_streaming_table_as_sql": {"name": "get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_streaming_table_as_sql", "macro_sql": "{% macro get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_streaming_table_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.615469, "supported_languages": null}, "macro.dbt_databricks.get_streaming_table_configuration_changes": {"name": "get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro_sql": "{% macro get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_streaming_table_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_streaming_table_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6158211, "supported_languages": null}, "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes": {"name": "databricks__get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes", "macro_sql": "\n\n{%- macro databricks__get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_streaming_table = adapter.get_relation_config(existing_relation) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = streaming_table.get_changeset(_existing_streaming_table) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6161752, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql": {"name": "databricks__get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {%- set alter_statement = get_alter_st_internal(relation, configuration_changes) -%}\n {%- set create_statement = get_create_st_internal(relation, configuration_changes, sql) -%}\n {%- set return_statements = [] -%}\n {%- if create_statement -%}\n {{ return_statements.append(create_statement) }}\n {%- endif -%}\n {%- if alter_statement -%}\n {{ return_statements.append(alter_statement) }}\n {%- endif -%}\n {% do return(return_statements) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_st_internal", "macro.dbt_databricks.get_create_st_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.616995, "supported_languages": null}, "macro.dbt_databricks.get_create_st_internal": {"name": "get_create_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_create_st_internal", "macro_sql": "{% macro get_create_st_internal(relation, configuration_changes, sql) %}\n {%- set partition_by = configuration_changes.changes[\"partition_by\"].partition_by -%}\n {%- set tblproperties = configuration_changes.changes[\"tblproperties\"].tblproperties -%}\n {%- set comment = configuration_changes.changes[\"comment\"].comment -%}\n CREATE OR REFRESH STREAMING TABLE {{ relation }}\n {% if partition_by -%}\n {{ get_create_sql_partition_by(partition_by) }}\n {%- endif %}\n {% if comment -%}\n {{ get_create_sql_comment(comment) }}\n {%- endif %}\n {% if tblproperties -%}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {%- endif %}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6175182, "supported_languages": null}, "macro.dbt_databricks.get_alter_st_internal": {"name": "get_alter_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_st_internal", "macro_sql": "{% macro get_alter_st_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n {%- if refresh and refresh.cron -%}\n ALTER STREAMING TABLE {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, False) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6177878, "supported_languages": null}, "macro.dbt_databricks.get_create_streaming_table_as_sql": {"name": "get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro_sql": "{% macro get_create_streaming_table_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_streaming_table_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618218, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql": {"name": "databricks__get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_create_streaming_table_as_sql(relation, sql) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = streaming_table.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = streaming_table.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = streaming_table.config[\"comment\"].comment -%}\n {%- set refresh = streaming_table.config[\"refresh\"] -%}\n\n CREATE STREAMING TABLE {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618762, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_comment": {"name": "get_create_sql_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/comment.sql", "original_file_path": "macros/relations/components/comment.sql", "unique_id": "macro.dbt_databricks.get_create_sql_comment", "macro_sql": "{%- macro get_create_sql_comment(comment) -%}\n{% if comment is string -%}\n COMMENT '{{ comment }}'\n{%- endif -%}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6189408, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_refresh_schedule": {"name": "get_create_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_create_sql_refresh_schedule", "macro_sql": "{% macro get_create_sql_refresh_schedule(cron, time_zone_value) %}\n {%- if cron -%}\n SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619374, "supported_languages": null}, "macro.dbt_databricks.get_alter_sql_refresh_schedule": {"name": "get_alter_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_alter_sql_refresh_schedule", "macro_sql": "{% macro get_alter_sql_refresh_schedule(cron, time_zone_value, is_altered) %}\n {%- if cron -%}\n {%- if is_altered -%}\n ALTER SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- else -%}\n ADD SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n {%- else -%}\n DROP SCHEDULE\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619678, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_partition_by": {"name": "get_create_sql_partition_by", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/partitioning.sql", "original_file_path": "macros/relations/components/partitioning.sql", "unique_id": "macro.dbt_databricks.get_create_sql_partition_by", "macro_sql": "{% macro get_create_sql_partition_by(partition_by) -%}\n{%- if partition_by -%}\n PARTITIONED BY ({%- for col in partition_by -%}{{ col }}{% if not loop.last %}, {% endif %}{%- endfor %})\n{%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619945, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_tblproperties": {"name": "get_create_sql_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/tblproperties.sql", "original_file_path": "macros/relations/components/tblproperties.sql", "unique_id": "macro.dbt_databricks.get_create_sql_tblproperties", "macro_sql": "{% macro get_create_sql_tblproperties(tblproperties) %}\n {%- if tblproperties and tblproperties|length>0 -%}\n TBLPROPERTIES (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}'{%- if not loop.last -%}, {% endif -%}\n {% endfor -%}\n )\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62031, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_table": {"name": "databricks__drop_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_table", "macro_sql": "{% macro databricks__drop_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.620424, "supported_languages": null}, "macro.dbt_databricks.databricks__create_table_as": {"name": "databricks__create_table_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_table_as", "macro_sql": "{% macro databricks__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {% if file_format == 'delta' %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ liquid_clustered_cols() }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ databricks__py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_databricks.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.partition_cols", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt_databricks.databricks__py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6221719, "supported_languages": null}, "macro.dbt_databricks.databricks__options_clause": {"name": "databricks__options_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__options_clause", "macro_sql": "{% macro databricks__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format', default='delta') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.623204, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_view": {"name": "databricks__drop_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_view", "macro_sql": "{% macro databricks__drop_view(relation) -%}\n drop view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6233542, "supported_languages": null}, "macro.dbt_databricks.databricks__create_view_as": {"name": "databricks__create_view_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_view_as", "macro_sql": "{% macro databricks__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n {% if query_columns %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n {{ tblproperties_clause() }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_databricks.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent", "macro.dbt_databricks.tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6240902, "supported_languages": null}, "macro.dbt_databricks.statement_with_staging_table": {"name": "statement_with_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.statement_with_staging_table", "macro_sql": "{% macro statement_with_staging_table(name=None, staging_table=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result, staging_table=staging_table) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6248162, "supported_languages": null}, "macro.dbt_databricks.databricks__dateadd": {"name": "databricks__dateadd", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_databricks.databricks__dateadd", "macro_sql": "{% macro databricks__dateadd(datepart, interval, from_date_or_timestamp) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampadd({{datepart}}, {{interval}}, {{from_date_or_timestamp}})\n {%- else -%}\n {{ spark__dateadd(datepart, interval, from_date_or_timestamp) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6251829, "supported_languages": null}, "macro.dbt_databricks.databricks__datediff": {"name": "databricks__datediff", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_databricks.databricks__datediff", "macro_sql": "{% macro databricks__datediff(first_date, second_date, datepart) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampdiff({{datepart}}, {{date_trunc(datepart, first_date)}}, {{date_trunc(datepart, second_date)}})\n {%- else -%}\n {{ spark__datediff(first_date, second_date, datepart) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.date_trunc", "macro.dbt_spark.spark__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6255841, "supported_languages": null}, "macro.dbt_databricks.databricks_copy_into": {"name": "databricks_copy_into", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/copy_into.sql", "original_file_path": "macros/adapters/copy_into.sql", "unique_id": "macro.dbt_databricks.databricks_copy_into", "macro_sql": "{% macro databricks_copy_into(\n target_table,\n source,\n file_format,\n expression_list=none,\n source_credential=none,\n source_encryption=none,\n validate=none,\n files=none,\n pattern=none,\n format_options=none,\n copy_options=none) -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=target.database,\n schema=target.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- set source_clause -%}\n {%- if expression_list -%}\n ( select {{ expression_list }} from '{{ source }}' )\n {%- else -%}\n '{{ source }}'\n {%- endif -%}\n {%- if source_credential or source_encryption %}\n WITH (\n {%- if source_credential %}\n credential (\n {%- for name in source_credential -%}\n '{{ name }}' = '{{ source_credential[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {%- if source_encryption %}\n encryption (\n {%- for name in source_encryption -%}\n '{{ name }}' = '{{ source_encryption[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n )\n {%- endif -%}\n {%- endset -%}\n\n {% set query %}\n copy into {{ target_relation }}\n from {{ source_clause }}\n fileformat = {{ file_format }}\n {% if validate -%} validate {{ validate }} {%- endif %}\n {% if files and pattern %}\n {{ exceptions.raise_compiler_error(\"You can only specify one of 'files' or 'pattern'\") }}\n {% endif %}\n {% if files -%}\n files = (\n {%- for file in files -%}\n '{{ file }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if pattern -%}\n pattern = '{{ pattern }}'\n {%- endif %}\n {% if format_options -%}\n format_options (\n {%- for key in format_options -%}\n '{{ key }}' = '{{ format_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if copy_options -%}\n copy_options (\n {%- for key in copy_options -%}\n '{{ key }}' = '{{ copy_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% endset %}\n\n {% do log(\"Running COPY INTO\" ~ adapter.redact_credentials(query), info=True) %}\n {% do run_query(query) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62939, "supported_languages": null}, "macro.dbt_databricks.databricks__make_temp_relation": {"name": "databricks__make_temp_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__make_temp_relation", "macro_sql": "{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp', as_table=False) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {%- if as_table -%}\n {% set tmp_relation = api.Relation.create(\n identifier=tmp_identifier,\n schema=base_relation.schema,\n database=base_relation.database,\n type='table') %}\n {%- else -%}\n {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}\n {%- endif -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.630194, "supported_languages": null}, "macro.dbt_databricks.databricks__get_or_create_relation": {"name": "databricks__get_or_create_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__get_or_create_relation", "macro_sql": "{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}\n {%- set target_relation = adapter.get_relation(\n database=database,\n schema=schema,\n identifier=identifier,\n needs_information=needs_information) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6306949, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog": {"name": "databricks__get_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog", "macro_sql": "{% macro databricks__get_catalog(information_schema, schemas) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633368, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations": {"name": "databricks__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations", "macro_sql": "{% macro databricks__get_catalog_relations(information_schema, relations) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63369, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_tables_sql": {"name": "databricks__get_catalog_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro_sql": "{% macro databricks__get_catalog_tables_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n lower(table_type) as table_type,\n comment as table_comment,\n table_owner,\n 'Last Modified' as `stats:last_modified:label`,\n last_altered as `stats:last_modified:value`,\n 'The timestamp for last update/change' as `stats:last_modified:description`,\n (last_altered is not null and table_type not ilike '%VIEW%') as `stats:last_modified:include`\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6338282, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_columns_sql": {"name": "databricks__get_catalog_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro_sql": "{% macro databricks__get_catalog_columns_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n column_name,\n ordinal_position as column_index,\n lower(data_type) as column_type,\n comment as column_comment\n from `system`.`information_schema`.`columns`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633953, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_results_sql": {"name": "databricks__get_catalog_results_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro_sql": "{% macro databricks__get_catalog_results_sql() -%}\n select *\n from tables\n join columns using (table_database, table_schema, table_name)\n order by column_index\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.634028, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql": {"name": "databricks__get_catalog_schemas_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_schemas_where_clause_sql(catalog, schemas) -%}\n where table_catalog = '{{ catalog|lower }}' and ({%- for relation in schemas -%}\n table_schema = '{{ relation[1]|lower }}'{%- if not loop.last %} or {% endif -%}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63427, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql": {"name": "databricks__get_catalog_relations_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_relations_where_clause_sql(catalog, relations) -%}\n where table_catalog = '{{ catalog|lower }}' and (\n {%- for relation in relations -%}\n {% if relation.schema and relation.identifier %}\n (\n table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n )\n {% elif relation.schema %}\n (\n table_schema = '{{ relation.schema|lower }}'\n )\n {% else %}\n {% do exceptions.raise_compiler_error(\n '`get_catalog_relations` requires a list of relations, each with a schema'\n ) %}\n {% endif %}\n\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6348171, "supported_languages": null}, "macro.dbt_databricks.current_catalog": {"name": "current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.current_catalog", "macro_sql": "{% macro current_catalog() -%}\n {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__current_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6351402, "supported_languages": null}, "macro.dbt_databricks.databricks__current_catalog": {"name": "databricks__current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__current_catalog", "macro_sql": "{% macro databricks__current_catalog() -%}\n {% call statement('current_catalog', fetch_result=True) %}\n select current_catalog()\n {% endcall %}\n {% do return(load_result('current_catalog').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635343, "supported_languages": null}, "macro.dbt_databricks.use_catalog": {"name": "use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.use_catalog", "macro_sql": "{% macro use_catalog(catalog) -%}\n {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__use_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6355011, "supported_languages": null}, "macro.dbt_databricks.databricks__use_catalog": {"name": "databricks__use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__use_catalog", "macro_sql": "{% macro databricks__use_catalog(catalog) -%}\n {% call statement() %}\n use catalog {{ adapter.quote(catalog)|lower }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635657, "supported_languages": null}, "macro.dbt_databricks.databricks__py_write_table": {"name": "databricks__py_write_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.databricks__py_write_table", "macro_sql": "{% macro databricks__py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\nimport pyspark\n\n{{ py_try_import('pyspark.sql.connect.dataframe', 'newer_pyspark_available') }}\n{{ py_try_import('pandas', 'pandas_available') }}\n{{ py_try_import('pyspark.pandas', 'pyspark_pandas_api_available') }}\n{{ py_try_import('databricks.koalas', 'koalas_available') }}\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\n\nif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n if pyspark_pandas_api_available:\n df = pyspark.pandas.frame.DataFrame(df)\n elif koalas_available:\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif newer_pyspark_available and isinstance(df, pyspark.sql.connect.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\nwriter = (\n df.write\n .mode(\"overwrite\")\n .option(\"overwriteSchema\", \"true\")\n{{ py_get_writer_options()|indent(8, True) }}\n)\n\nwriter.saveAsTable(\"{{ target_relation }}\")\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.py_try_import", "macro.dbt_databricks.py_get_writer_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6376588, "supported_languages": null}, "macro.dbt_databricks.py_get_writer_options": {"name": "py_get_writer_options", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_get_writer_options", "macro_sql": "\n\n{%- macro py_get_writer_options() -%}\n{%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n{%- set file_format = config.get('file_format', validator=validation.any[basestring])|default('delta', true) -%}\n{%- set partition_by = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n{%- set liquid_clustered_by = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set clustered_by = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n.format(\"{{ file_format }}\")\n{%- if location_root is not none %}\n{%- set identifier = model['alias'] %}\n{%- if is_incremental() %}\n{%- set identifier = identifier + '__dbt_tmp' %}\n{%- endif %}\n.option(\"path\", \"{{ location_root }}/{{ identifier }}\")\n{%- endif -%}\n{%- if partition_by is not none -%}\n {%- if partition_by is string -%}\n {%- set partition_by = [partition_by] -%}\n {%- endif %}\n.partitionBy({{ partition_by }})\n{%- endif -%}\n{%- if liquid_clustered_by and not is_incremental() -%}\n {%- if liquid_clustered_by is string -%}\n {%- set liquid_clustered_by = [liquid_clustered_by] -%}\n {%- endif %}\n.clusterBy({{ liquid_clustered_by }})\n{%- endif -%}\n{%- if (clustered_by is not none) and (buckets is not none) -%}\n {%- if clustered_by is string -%}\n {%- set clustered_by = [clustered_by] -%}\n {%- endif %}\n.bucketBy({{ buckets }}, {{ clustered_by }})\n{%- endif -%}\n{% endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.is_incremental"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6390772, "supported_languages": null}, "macro.dbt_databricks.py_try_import": {"name": "py_try_import", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_try_import", "macro_sql": "{% macro py_try_import(library, var_name) -%}\n# make sure {{ library }} exists before using it\ntry:\n import {{ library }}\n {{ var_name }} = True\nexcept ImportError:\n {{ var_name }} = False\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.639234, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_comment": {"name": "databricks__alter_column_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_comment", "macro_sql": "{% macro databricks__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', default='delta') in ['delta', 'hudi'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641151, "supported_languages": null}, "macro.dbt_databricks.alter_table_comment": {"name": "alter_table_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_table_comment", "macro_sql": "{% macro alter_table_comment(relation, model) %}\n {% set comment_query %}\n comment on table {{ relation|lower }} is '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endset %}\n {% do run_query(comment_query) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641404, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments": {"name": "get_columns_comments", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_columns_comments", "macro_sql": "{% macro get_columns_comments(relation) -%}\n {% call statement('get_columns_comments', fetch_result=True) -%}\n describe table {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('get_columns_comments').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6416378, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_docs": {"name": "databricks__persist_docs", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__persist_docs", "macro_sql": "{% macro databricks__persist_docs(relation, model, for_relation, for_columns) -%}\n {%- if for_relation and config.persist_relation_docs() and model.description %}\n {% do alter_table_comment(relation, model) %}\n {% endif %}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {%- set existing_columns = adapter.get_columns_in_relation(relation) -%}\n {%- set columns_to_persist_docs = adapter.get_persist_doc_columns(existing_columns, model.columns) -%}\n {% do alter_column_comment(relation, columns_to_persist_docs) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.alter_table_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642109, "supported_languages": null}, "macro.dbt_databricks.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642476, "supported_languages": null}, "macro.dbt_databricks.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642708, "supported_languages": null}, "macro.dbt_databricks.databricks__list_relations_without_caching": {"name": "databricks__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__list_relations_without_caching", "macro_sql": "{% macro databricks__list_relations_without_caching(schema_relation) %}\n {{ return(adapter.get_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6461172, "supported_languages": null}, "macro.dbt_databricks.show_table_extended": {"name": "show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_table_extended", "macro_sql": "{% macro show_table_extended(schema_relation) %}\n {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_table_extended"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646267, "supported_languages": null}, "macro.dbt_databricks.databricks__show_table_extended": {"name": "databricks__show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_table_extended", "macro_sql": "{% macro databricks__show_table_extended(schema_relation) %}\n {% call statement('show_table_extended', fetch_result=True) -%}\n show table extended in {{ schema_relation.without_identifier()|lower }} like '{{ schema_relation.identifier|lower }}'\n {% endcall %}\n\n {% do return(load_result('show_table_extended').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646537, "supported_languages": null}, "macro.dbt_databricks.show_tables": {"name": "show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_tables", "macro_sql": "{% macro show_tables(relation) %}\n {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_tables"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646763, "supported_languages": null}, "macro.dbt_databricks.databricks__show_tables": {"name": "databricks__show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_tables", "macro_sql": "{% macro databricks__show_tables(relation) %}\n {% call statement('show_tables', fetch_result=True) -%}\n show tables in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646989, "supported_languages": null}, "macro.dbt_databricks.show_views": {"name": "show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_views", "macro_sql": "{% macro show_views(relation) %}\n {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_views"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647151, "supported_languages": null}, "macro.dbt_databricks.databricks__show_views": {"name": "databricks__show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_views", "macro_sql": "{% macro databricks__show_views(relation) %}\n {% call statement('show_views', fetch_result=True) -%}\n show views in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_views').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647396, "supported_languages": null}, "macro.dbt_databricks.databricks__get_relation_last_modified": {"name": "databricks__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__get_relation_last_modified", "macro_sql": "{% macro databricks__get_relation_last_modified(information_schema, relations) -%}\n\n {%- call statement('last_modified', fetch_result=True) -%}\n {% if information_schema.is_hive_metastore() %}\n {%- for relation in relations -%}\n select '{{ relation.schema }}' as schema,\n '{{ relation.identifier }}' as identifier,\n max(timestamp) as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from (describe history {{ relation.schema|lower }}.{{ relation.identifier|lower }})\n {% if not loop.last %}\n union all\n {% endif %}\n {%- endfor -%}\n {% else %}\n select table_schema as schema,\n table_name as identifier,\n last_altered as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}' and\n (\n {%- for relation in relations -%}\n (table_schema = '{{ relation.schema|lower }}' and\n table_name = '{{ relation.identifier|lower }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n {% endif %}\n {%- endcall -%}\n\n {{ return(load_result('last_modified')) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6481018, "supported_languages": null}, "macro.dbt_databricks.get_view_description": {"name": "get_view_description", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_view_description", "macro_sql": "{% macro get_view_description(relation) %}\n {% call statement('get_view_description', fetch_result=True) -%}\n select *\n from `system`.`information_schema`.`views`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n {%- endcall -%}\n\n {% do return(load_result('get_view_description').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648402, "supported_languages": null}, "macro.dbt_databricks.get_uc_tables": {"name": "get_uc_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_uc_tables", "macro_sql": "{% macro get_uc_tables(relation) %}\n {% call statement('get_uc_tables', fetch_result=True) -%}\n select\n table_name,\n if(table_type in ('EXTERNAL', 'MANAGED', 'MANAGED_SHALLOW_CLONE', 'EXTERNAL_SHALLOW_CLONE'), 'table', lower(table_type)) as table_type,\n lower(data_source_format) as file_format,\n table_owner\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n {% if relation.identifier %}\n and table_name = '{{ relation.identifier|lower }}'\n {% endif %}\n {% endcall %}\n\n {% do return(load_result('get_uc_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648753, "supported_languages": null}, "macro.dbt_spark.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() %}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663456, "supported_languages": null}, "macro.dbt_spark.spark__tblproperties_clause": {"name": "spark__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__tblproperties_clause", "macro_sql": "{% macro spark__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663767, "supported_languages": null}, "macro.dbt_spark.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {{ return(adapter.dispatch('file_format_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__file_format_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663905, "supported_languages": null}, "macro.dbt_spark.spark__file_format_clause": {"name": "spark__file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__file_format_clause", "macro_sql": "{% macro spark__file_format_clause() %}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n {%- if file_format is not none %}\n using {{ file_format }}\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6641312, "supported_languages": null}, "macro.dbt_spark.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.location_clause", "macro_sql": "{% macro location_clause() %}\n {{ return(adapter.dispatch('location_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__location_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664276, "supported_languages": null}, "macro.dbt_spark.spark__location_clause": {"name": "spark__location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__location_clause", "macro_sql": "{% macro spark__location_clause() %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664563, "supported_languages": null}, "macro.dbt_spark.options_clause": {"name": "options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.options_clause", "macro_sql": "{% macro options_clause() -%}\n {{ return(adapter.dispatch('options_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__options_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6646988, "supported_languages": null}, "macro.dbt_spark.spark__options_clause": {"name": "spark__options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__options_clause", "macro_sql": "{% macro spark__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6656451, "supported_languages": null}, "macro.dbt_spark.comment_clause": {"name": "comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.comment_clause", "macro_sql": "{% macro comment_clause() %}\n {{ return(adapter.dispatch('comment_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__comment_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.665798, "supported_languages": null}, "macro.dbt_spark.spark__comment_clause": {"name": "spark__comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__comment_clause", "macro_sql": "{% macro spark__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endif %}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666194, "supported_languages": null}, "macro.dbt_spark.partition_cols": {"name": "partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.partition_cols", "macro_sql": "{% macro partition_cols(label, required=false) %}\n {{ return(adapter.dispatch('partition_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666384, "supported_languages": null}, "macro.dbt_spark.spark__partition_cols": {"name": "spark__partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__partition_cols", "macro_sql": "{% macro spark__partition_cols(label, required=false) %}\n {%- set cols = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666821, "supported_languages": null}, "macro.dbt_spark.clustered_cols": {"name": "clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.clustered_cols", "macro_sql": "{% macro clustered_cols(label, required=false) %}\n {{ return(adapter.dispatch('clustered_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__clustered_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667007, "supported_languages": null}, "macro.dbt_spark.spark__clustered_cols": {"name": "spark__clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__clustered_cols", "macro_sql": "{% macro spark__clustered_cols(label, required=false) %}\n {%- set cols = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n {%- if (cols is not none) and (buckets is not none) %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n ) into {{ buckets }} buckets\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6675909, "supported_languages": null}, "macro.dbt_spark.fetch_tbl_properties": {"name": "fetch_tbl_properties", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.fetch_tbl_properties", "macro_sql": "{% macro fetch_tbl_properties(relation) -%}\n {% call statement('list_properties', fetch_result=True) -%}\n SHOW TBLPROPERTIES {{ relation }}\n {% endcall %}\n {% do return(load_result('list_properties').table) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667808, "supported_languages": null}, "macro.dbt_spark.create_temporary_view": {"name": "create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.create_temporary_view", "macro_sql": "{% macro create_temporary_view(relation, compiled_code) -%}\n {{ return(adapter.dispatch('create_temporary_view', 'dbt')(relation, compiled_code)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__create_temporary_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667983, "supported_languages": null}, "macro.dbt_spark.spark__create_temporary_view": {"name": "spark__create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_temporary_view", "macro_sql": "{% macro spark__create_temporary_view(relation, compiled_code) -%}\n create or replace temporary view {{ relation }} as\n {{ compiled_code }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668092, "supported_languages": null}, "macro.dbt_spark.spark__create_table_as": {"name": "spark__create_table_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_table_as", "macro_sql": "{%- macro spark__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'iceberg'] %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ tblproperties_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_spark.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.tblproperties_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_spark.py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668963, "supported_languages": null}, "macro.dbt_spark.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669142, "supported_languages": null}, "macro.dbt_spark.spark__persist_constraints": {"name": "spark__persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_constraints", "macro_sql": "{% macro spark__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config.enforced and config.get('file_format', 'delta') == 'delta' %}\n {% do alter_table_add_constraints(relation, model.constraints) %}\n {% do alter_column_set_constraints(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.alter_table_add_constraints", "macro.dbt_spark.alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669481, "supported_languages": null}, "macro.dbt_spark.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669651, "supported_languages": null}, "macro.dbt_spark.spark__alter_table_add_constraints": {"name": "spark__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_table_add_constraints", "macro_sql": "{% macro spark__alter_table_add_constraints(relation, constraints) %}\n {% for constraint in constraints %}\n {% if constraint.type == 'check' and not is_incremental() %}\n {%- set constraint_hash = local_md5(column_name ~ \";\" ~ constraint.expression ~ \";\" ~ loop.index) -%}\n {% call statement() %}\n alter table {{ relation }} add constraint {{ constraint.name if constraint.name else constraint_hash }} check ({{ constraint.expression }});\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670092, "supported_languages": null}, "macro.dbt_spark.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67027, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_set_constraints": {"name": "spark__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_set_constraints", "macro_sql": "{% macro spark__alter_column_set_constraints(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set constraints = column_dict[column_name]['constraints'] %}\n {% for constraint in constraints %}\n {% if constraint.type != 'not_null' %}\n {{ exceptions.warn('Invalid constraint for column ' ~ column_name ~ '. Only `not_null` is supported.') }}\n {% else %}\n {% set quoted_name = adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name %}\n {% call statement() %}\n alter table {{ relation }} change column {{ quoted_name }} set not null {{ constraint.expression or \"\" }};\n {% endcall %}\n {% endif %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670861, "supported_languages": null}, "macro.dbt_spark.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671293, "supported_languages": null}, "macro.dbt_spark.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671527, "supported_languages": null}, "macro.dbt_spark.spark__create_view_as": {"name": "spark__create_view_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_view_as", "macro_sql": "{% macro spark__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_spark.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6719759, "supported_languages": null}, "macro.dbt_spark.spark__create_schema": {"name": "spark__create_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_schema", "macro_sql": "{% macro spark__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{relation}}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6721191, "supported_languages": null}, "macro.dbt_spark.spark__drop_schema": {"name": "spark__drop_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_schema", "macro_sql": "{% macro spark__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672259, "supported_languages": null}, "macro.dbt_spark.get_columns_in_relation_raw": {"name": "get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_columns_in_relation_raw", "macro_sql": "{% macro get_columns_in_relation_raw(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation_raw', 'dbt')(relation)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation_raw"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672405, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation_raw": {"name": "spark__get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation_raw", "macro_sql": "{% macro spark__get_columns_in_relation_raw(relation) -%}\n {% call statement('get_columns_in_relation_raw', fetch_result=True) %}\n describe extended {{ relation }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation_raw').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672622, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation": {"name": "spark__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation", "macro_sql": "{% macro spark__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n describe extended {{ relation.include(schema=(schema is not none)) }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672888, "supported_languages": null}, "macro.dbt_spark.spark__list_relations_without_caching": {"name": "spark__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_relations_without_caching", "macro_sql": "{% macro spark__list_relations_without_caching(relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n show table extended in {{ relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673117, "supported_languages": null}, "macro.dbt_spark.list_relations_show_tables_without_caching": {"name": "list_relations_show_tables_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.list_relations_show_tables_without_caching", "macro_sql": "{% macro list_relations_show_tables_without_caching(schema_relation) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('list_relations_without_caching_show_tables', fetch_result=True) -%}\n show tables in {{ schema_relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching_show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6733599, "supported_languages": null}, "macro.dbt_spark.describe_table_extended_without_caching": {"name": "describe_table_extended_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.describe_table_extended_without_caching", "macro_sql": "{% macro describe_table_extended_without_caching(table_name) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('describe_table_extended_without_caching', fetch_result=True) -%}\n describe extended {{ table_name }}\n {% endcall %}\n {% do return(load_result('describe_table_extended_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6736, "supported_languages": null}, "macro.dbt_spark.spark__list_schemas": {"name": "spark__list_schemas", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_schemas", "macro_sql": "{% macro spark__list_schemas(database) -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n show databases\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673813, "supported_languages": null}, "macro.dbt_spark.spark__rename_relation": {"name": "spark__rename_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__rename_relation", "macro_sql": "{% macro spark__rename_relation(from_relation, to_relation) -%}\n {% call statement('rename_relation') -%}\n {% if not from_relation.type %}\n {% do exceptions.raise_database_error(\"Cannot rename a relation with a blank type: \" ~ from_relation.identifier) %}\n {% elif from_relation.type in ('table') %}\n alter table {{ from_relation }} rename to {{ to_relation }}\n {% elif from_relation.type == 'view' %}\n alter view {{ from_relation }} rename to {{ to_relation }}\n {% else %}\n {% do exceptions.raise_database_error(\"Unknown type '\" ~ from_relation.type ~ \"' for relation: \" ~ from_relation.identifier) %}\n {% endif %}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674293, "supported_languages": null}, "macro.dbt_spark.spark__drop_relation": {"name": "spark__drop_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_relation", "macro_sql": "{% macro spark__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674477, "supported_languages": null}, "macro.dbt_spark.spark__generate_database_name": {"name": "spark__generate_database_name", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__generate_database_name", "macro_sql": "{% macro spark__generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(None) %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674613, "supported_languages": null}, "macro.dbt_spark.spark__persist_docs": {"name": "spark__persist_docs", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_docs", "macro_sql": "{% macro spark__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do alter_column_comment(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674838, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_comment": {"name": "spark__alter_column_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_comment", "macro_sql": "{% macro spark__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'hudi', 'iceberg'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n {% if relation.is_iceberg %}\n alter table {{ relation }} alter column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% else %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endif %}\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.675686, "supported_languages": null}, "macro.dbt_spark.spark__make_temp_relation": {"name": "spark__make_temp_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__make_temp_relation", "macro_sql": "{% macro spark__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(path = {\n \"identifier\": tmp_identifier\n }) -%}\n\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6760252, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_type": {"name": "spark__alter_column_type", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_type", "macro_sql": "{% macro spark__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ column_name }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67621, "supported_languages": null}, "macro.dbt_spark.spark__alter_relation_add_remove_columns": {"name": "spark__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_relation_add_remove_columns", "macro_sql": "{% macro spark__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if remove_columns %}\n {% if relation.is_delta %}\n {% set platform_name = 'Delta Lake' %}\n {% elif relation.is_iceberg %}\n {% set platform_name = 'Iceberg' %}\n {% else %}\n {% set platform_name = 'Apache Spark' %}\n {% endif %}\n {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}\n {% endif %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% if add_columns %} add columns {% endif %}\n {% for column in add_columns %}\n {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.676926, "supported_languages": null}, "macro.dbt_spark.spark__copy_grants": {"name": "spark__copy_grants", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__copy_grants", "macro_sql": "{% macro spark__copy_grants() %}\n\n {% if config.materialized == 'view' %}\n {#-- Spark views don't copy grants when they're replaced --#}\n {{ return(False) }}\n\n {% else %}\n {#-- This depends on how we're replacing the table, which depends on its file format\n -- Just play it safe by assuming that grants have been copied over, and need to be checked / possibly revoked\n -- We can make this more efficient in the future\n #}\n {{ return(True) }}\n\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677521, "supported_languages": null}, "macro.dbt_spark.spark__get_grant_sql": {"name": "spark__get_grant_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_grant_sql", "macro_sql": "\n\n\n{%- macro spark__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6777, "supported_languages": null}, "macro.dbt_spark.spark__get_revoke_sql": {"name": "spark__get_revoke_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_revoke_sql", "macro_sql": "\n\n\n{%- macro spark__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677873, "supported_languages": null}, "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement": {"name": "spark__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n\n{%- macro spark__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(False) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677966, "supported_languages": null}, "macro.dbt_spark.spark__call_dcl_statements": {"name": "spark__call_dcl_statements", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__call_dcl_statements", "macro_sql": "{% macro spark__call_dcl_statements(dcl_statement_list) %}\n {% for dcl_statement in dcl_statement_list %}\n {% call statement('grant_or_revoke') %}\n {{ dcl_statement }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.678161, "supported_languages": null}, "macro.dbt_spark.spark__get_binding_char": {"name": "spark__get_binding_char", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__get_binding_char", "macro_sql": "{% macro spark__get_binding_char() %}\n {{ return('?' if target.method == 'odbc' else '%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.679692, "supported_languages": null}, "macro.dbt_spark.spark__reset_csv_table": {"name": "spark__reset_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__reset_csv_table", "macro_sql": "{% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {{ adapter.drop_relation(old_relation) }}\n {% endif %}\n {% set sql = create_csv_table(model, agate_table) %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6799822, "supported_languages": null}, "macro.dbt_spark.spark__load_csv_rows": {"name": "spark__load_csv_rows", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__load_csv_rows", "macro_sql": "{% macro spark__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.681087, "supported_languages": null}, "macro.dbt_spark.spark__create_csv_table": {"name": "spark__create_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__create_csv_table", "macro_sql": "{% macro spark__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6820521, "supported_languages": null}, "macro.dbt_spark.materialization_view_spark": {"name": "materialization_view_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_spark.materialization_view_spark", "macro_sql": "{% materialization view, adapter='spark' -%}\n {{ return(create_or_replace_view()) }}\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.create_or_replace_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.682219, "supported_languages": ["sql"]}, "macro.dbt_spark.materialization_table_spark": {"name": "materialization_table_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.materialization_table_spark", "macro_sql": "{% materialization table, adapter = 'spark', supported_languages=['sql', 'python'] %}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation is not none %}\n {% set is_delta = (old_relation.is_delta and config.get('file_format', validator=validation.any[basestring]) == 'delta') %}\n {% set is_iceberg = (old_relation.is_iceberg and config.get('file_format', validator=validation.any[basestring]) == 'iceberg') %}\n {% set old_relation_type = old_relation.type %}\n {% else %}\n {% set is_delta = false %}\n {% set is_iceberg = false %}\n {% set old_relation_type = target_relation.type %}\n {% endif %}\n\n {% if not is_delta and not is_iceberg %}\n {% set existing_relation = target_relation %}\n {{ adapter.drop_relation(existing_relation.incorporate(type=old_relation_type)) }}\n {% endif %}\n\n -- build model\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt_spark.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6857522, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.py_write_table": {"name": "py_write_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_write_table", "macro_sql": "{% macro py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\n# make sure pyspark exists in the namepace, for 7.3.x-scala2.12 it does not exist\nimport pyspark\n# make sure pandas exists before using it\ntry:\n import pandas\n pandas_available = True\nexcept ImportError:\n pandas_available = False\n\n# make sure pyspark.pandas exists before using it\ntry:\n import pyspark.pandas\n pyspark_pandas_api_available = True\nexcept ImportError:\n pyspark_pandas_api_available = False\n\n# make sure databricks.koalas exists before using it\ntry:\n import databricks.koalas\n koalas_available = True\nexcept ImportError:\n koalas_available = False\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\nif pyspark_pandas_api_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = pyspark.pandas.frame.DataFrame(df)\nelif koalas_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\ndf.write.mode(\"overwrite\").format(\"{{ config.get('file_format', 'delta') }}\").option(\"overwriteSchema\", \"true\").saveAsTable(\"{{ target_relation }}\")\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686034, "supported_languages": null}, "macro.dbt_spark.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n# how to execute python model in notebook\n# dbt = dbtObj(spark.table)\n# df = model(dbt, spark)\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686104, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_hash_arguments": {"name": "spark__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_hash_arguments", "macro_sql": "{% macro spark__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as string ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.692856, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_string_as_time": {"name": "spark__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_string_as_time", "macro_sql": "{% macro spark__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"to_timestamp('\" ~ timestamp ~ \"')\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693005, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_merge_sql": {"name": "spark__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_merge_sql", "macro_sql": "{% macro spark__snapshot_merge_sql(target, source, insert_cols) -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {% if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {% else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {% endif %}\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert *\n ;\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693233, "supported_languages": null}, "macro.dbt_spark.spark_build_snapshot_staging_table": {"name": "spark_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark_build_snapshot_staging_table", "macro_sql": "{% macro spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {% if target_relation.is_iceberg %}\n {# iceberg catalog does not support create view, but regular spark does. We removed the catalog and schema #}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=none,\n database=none,\n type='view') -%}\n {% else %}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=none,\n type='view') -%}\n {% endif %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_view_as(tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693888, "supported_languages": null}, "macro.dbt_spark.spark__post_snapshot": {"name": "spark__post_snapshot", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__post_snapshot", "macro_sql": "{% macro spark__post_snapshot(staging_relation) %}\n {% do adapter.drop_relation(staging_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.694009, "supported_languages": null}, "macro.dbt_spark.spark__create_columns": {"name": "spark__create_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__create_columns", "macro_sql": "{% macro spark__create_columns(relation, columns) %}\n {% if columns|length > 0 %}\n {% call statement() %}\n alter table {{ relation }} add columns (\n {% for column in columns %}\n `{{ column.name }}` {{ column.data_type }} {{- ',' if not loop.last -}}\n {% endfor %}\n );\n {% endcall %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6943328, "supported_languages": null}, "macro.dbt_spark.materialization_snapshot_spark": {"name": "materialization_snapshot_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.materialization_snapshot_spark", "macro_sql": "{% materialization snapshot, adapter='spark' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'parquet') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=none,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if file_format not in ['delta', 'iceberg', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_iceberg and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.schema) %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.create_schema", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6975749, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__can_clone_table": {"name": "spark__can_clone_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__can_clone_table", "macro_sql": "{% macro spark__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6994078, "supported_languages": null}, "macro.dbt_spark.spark__create_or_replace_clone": {"name": "spark__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__create_or_replace_clone", "macro_sql": "{% macro spark__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.699519, "supported_languages": null}, "macro.dbt_spark.materialization_clone_spark": {"name": "materialization_clone_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.materialization_clone_spark", "macro_sql": "\n\n{%- materialization clone, adapter='spark' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if file_format != 'delta' -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n shallow clone requires file_format be set to 'delta'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {%- elif other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.70128, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__get_merge_update_columns": {"name": "spark__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/column_helpers.sql", "original_file_path": "macros/materializations/incremental/column_helpers.sql", "unique_id": "macro.dbt_spark.spark__get_merge_update_columns", "macro_sql": "{% macro spark__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.702069, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_file_format": {"name": "dbt_spark_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_file_format", "macro_sql": "{% macro dbt_spark_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'iceberg', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703007, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy": {"name": "dbt_spark_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'append', 'merge', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_merge_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via endpoint\n Use the 'append' or 'merge' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'iceberg', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_merge_msg) %}\n {% endif %}\n {% if raw_strategy == 'insert_overwrite' and target.endpoint %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_endpoint_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703701, "supported_languages": null}, "macro.dbt_spark.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation, existing_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n {% if existing_relation.is_iceberg %}\n {# removed table from statement for iceberg #}\n insert overwrite {{ target_relation }}\n {# removed partition_cols for iceberg as well #}\n {% else %}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {% endif %}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705375, "supported_languages": null}, "macro.dbt_spark.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert into table {{ target_relation }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705639, "supported_languages": null}, "macro.dbt_spark.spark__get_merge_sql": {"name": "spark__get_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.spark__get_merge_sql", "macro_sql": "{% macro spark__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n when matched then update set\n {% if update_columns -%}{%- for column_name in update_columns %}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {%- else %} * {% endif %}\n\n when not matched then insert *\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.706816, "supported_languages": null}, "macro.dbt_spark.dbt_spark_get_incremental_sql": {"name": "dbt_spark_get_incremental_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro_sql": "{% macro dbt_spark_get_incremental_sql(strategy, source, target, existing, unique_key, incremental_predicates) %}\n {%- if strategy == 'append' -%}\n {#-- insert new records into existing table, without updating or overwriting #}\n {{ get_insert_into_sql(source, target) }}\n {%- elif strategy == 'insert_overwrite' -%}\n {#-- insert statements don't like CTEs, so support them via a temp view #}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'merge' -%}\n {#-- merge all columns for datasources which implement MERGE INTO (e.g. databricks, iceberg) - schema changes are handled for us #}\n {{ get_merge_sql(target, source, unique_key, dest_columns=none, incremental_predicates=incremental_predicates) }}\n {%- else -%}\n {% set no_sql_for_strategy_msg -%}\n No known SQL for the incremental strategy provided: {{ strategy }}\n {%- endset %}\n {%- do exceptions.raise_compiler_error(no_sql_for_strategy_msg) -%}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_insert_into_sql", "macro.dbt_spark.get_insert_overwrite_sql", "macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.707412, "supported_languages": null}, "macro.dbt_spark.materialization_incremental_spark": {"name": "materialization_incremental_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_spark.materialization_incremental_spark", "macro_sql": "{% materialization incremental, adapter='spark', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='parquet') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'append' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set file_format = dbt_spark_validate_get_file_format(raw_file_format) -%}\n {%- set strategy = dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}\n {%- set target_relation = this -%}\n {%- set existing_relation = load_relation(this) -%}\n {%- set tmp_relation = make_temp_relation(this) -%}\n\n {#-- for SQL model we will create temp view that doesn't have database and schema --#}\n {%- if language == 'sql'-%}\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode --#}\n {%- if strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- elif existing_relation.is_view or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- else -%}\n {#-- Relation must be merged --#}\n {%- call statement('create_tmp_relation', language=language) -%}\n {{ create_table_as(True, tmp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, tmp_relation, existing_relation) -%}\n {%- call statement('main') -%}\n {{ dbt_spark_get_incremental_sql(strategy, tmp_relation, target_relation, existing_relation, unique_key, incremental_predicates) }}\n {%- endcall -%}\n {%- if language == 'python' -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why do neither drop_relation or adapter.drop_relation work here?!\n --#}\n {% call statement('drop_relation') -%}\n drop table if exists {{ tmp_relation }}\n {%- endcall %}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_spark.dbt_spark_validate_get_file_format", "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_spark.persist_constraints", "macro.dbt.should_full_refresh", "macro.dbt.process_schema_changes", "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710429, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.spark__concat": {"name": "spark__concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt_spark.spark__concat", "macro_sql": "{% macro spark__concat(fields) -%}\n concat({{ fields|join(', ') }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710591, "supported_languages": null}, "macro.dbt_spark.assert_not_null": {"name": "assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.assert_not_null", "macro_sql": "{% macro assert_not_null(function, arg) -%}\n {{ return(adapter.dispatch('assert_not_null', 'dbt')(function, arg)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710856, "supported_languages": null}, "macro.dbt_spark.spark__assert_not_null": {"name": "spark__assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.spark__assert_not_null", "macro_sql": "{% macro spark__assert_not_null(function, arg) %}\n\n coalesce({{function}}({{arg}}), nvl2({{function}}({{arg}}), assert_true({{function}}({{arg}}) is not null), null))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.711019, "supported_languages": null}, "macro.dbt_spark.spark__dateadd": {"name": "spark__dateadd", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_spark.spark__dateadd", "macro_sql": "{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {%- set clock_component -%}\n {# make sure the dates + timestamps are real, otherwise raise an error asap #}\n to_unix_timestamp({{ assert_not_null('to_timestamp', from_date_or_timestamp) }})\n - to_unix_timestamp({{ assert_not_null('date', from_date_or_timestamp) }})\n {%- endset -%}\n\n {%- if datepart in ['day', 'week'] -%}\n\n {%- set multiplier = 7 if datepart == 'week' else 1 -%}\n\n to_timestamp(\n to_unix_timestamp(\n date_add(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ['month', 'quarter', 'year'] -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'month' -%} 1\n {%- elif datepart == 'quarter' -%} 3\n {%- elif datepart == 'year' -%} 12\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp(\n add_months(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n {{ assert_not_null('to_unix_timestamp', from_date_or_timestamp) }}\n + cast({{interval}} * {{multiplier}} as int)\n )\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro dateadd not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713269, "supported_languages": null}, "macro.dbt_spark.spark__current_timestamp": {"name": "spark__current_timestamp", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/timestamps.sql", "original_file_path": "macros/utils/timestamps.sql", "unique_id": "macro.dbt_spark.spark__current_timestamp", "macro_sql": "{% macro spark__current_timestamp() -%}\n current_timestamp()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713365, "supported_languages": null}, "macro.dbt_spark.spark__escape_single_quotes": {"name": "spark__escape_single_quotes", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt_spark.spark__escape_single_quotes", "macro_sql": "{% macro spark__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"\\\\'\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713526, "supported_languages": null}, "macro.dbt_spark.spark__listagg": {"name": "spark__listagg", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_spark.spark__listagg", "macro_sql": "{% macro spark__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if order_by_clause %}\n {{ exceptions.warn(\"order_by_clause is not supported for listagg on Spark/Databricks\") }}\n {% endif %}\n\n {% set collect_list %} collect_list({{ measure }}) {% endset %}\n\n {% set limited %} slice({{ collect_list }}, 1, {{ limit_num }}) {% endset %}\n\n {% set collected = limited if limit_num else collect_list %}\n\n {% set final %} array_join({{ collected }}, {{ delimiter_text }}) {% endset %}\n\n {% do return(final) %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.714139, "supported_languages": null}, "macro.dbt_spark.spark__datediff": {"name": "spark__datediff", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_spark.spark__datediff", "macro_sql": "{% macro spark__datediff(first_date, second_date, datepart) %}\n\n {%- if datepart in ['day', 'week', 'month', 'quarter', 'year'] -%}\n\n {# make sure the dates are real, otherwise raise an error asap #}\n {% set first_date = assert_not_null('date', first_date) %}\n {% set second_date = assert_not_null('date', second_date) %}\n\n {%- endif -%}\n\n {%- if datepart == 'day' -%}\n\n datediff({{second_date}}, {{first_date}})\n\n {%- elif datepart == 'week' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(datediff({{second_date}}, {{first_date}})/7)\n else ceil(datediff({{second_date}}, {{first_date}})/7)\n end\n\n -- did we cross a week boundary (Sunday)?\n + case\n when {{first_date}} < {{second_date}} and dayofweek({{second_date}}) < dayofweek({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofweek({{second_date}}) > dayofweek({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'month' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}})))\n else ceil(months_between(date({{second_date}}), date({{first_date}})))\n end\n\n -- did we cross a month boundary?\n + case\n when {{first_date}} < {{second_date}} and dayofmonth({{second_date}}) < dayofmonth({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofmonth({{second_date}}) > dayofmonth({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'quarter' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}}))/3)\n else ceil(months_between(date({{second_date}}), date({{first_date}}))/3)\n end\n\n -- did we cross a quarter boundary?\n + case\n when {{first_date}} < {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n < (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then 1\n when {{first_date}} > {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n > (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then -1\n else 0 end\n\n {%- elif datepart == 'year' -%}\n\n year({{second_date}}) - year({{first_date}})\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set divisor -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n case when {{first_date}} < {{second_date}}\n then ceil((\n {# make sure the timestamps are real, otherwise raise an error asap #}\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n else floor((\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n end\n\n {% if datepart == 'millisecond' %}\n + cast(date_format({{second_date}}, 'SSS') as int)\n - cast(date_format({{first_date}}, 'SSS') as int)\n {% endif %}\n\n {% if datepart == 'microsecond' %}\n {% set capture_str = '[0-9]{4}-[0-9]{2}-[0-9]{2}.[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]{6})' %}\n -- Spark doesn't really support microseconds, so this is a massive hack!\n -- It will only work if the timestamp-string is of the format\n -- 'yyyy-MM-dd-HH mm.ss.SSSSSS'\n + cast(regexp_extract({{second_date}}, '{{capture_str}}', 1) as int)\n - cast(regexp_extract({{first_date}}, '{{capture_str}}', 1) as int)\n {% endif %}\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro datediff not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7200809, "supported_languages": null}, "macro.dbt_spark.spark__safe_cast": {"name": "spark__safe_cast", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.spark__safe_cast", "macro_sql": "{% macro spark__safe_cast(field, type) %}\n{%- set field_clean = field.strip('\"').strip(\"'\") if (cast_from_string_unsupported_for(type) and field is string) else field -%}\ncast({{field_clean}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.cast_from_string_unsupported_for"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7204401, "supported_languages": null}, "macro.dbt_spark.cast_from_string_unsupported_for": {"name": "cast_from_string_unsupported_for", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.cast_from_string_unsupported_for", "macro_sql": "{% macro cast_from_string_unsupported_for(type) %}\n {{ return(type.lower().startswith('struct') or type.lower().startswith('array') or type.lower().startswith('map')) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720673, "supported_languages": null}, "macro.dbt_spark.spark__any_value": {"name": "spark__any_value", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_spark.spark__any_value", "macro_sql": "{% macro spark__any_value(expression) -%}\n {#-- return any value (non-deterministic) --#}\n first({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720806, "supported_languages": null}, "macro.dbt_spark.spark__array_concat": {"name": "spark__array_concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt_spark.spark__array_concat", "macro_sql": "{% macro spark__array_concat(array_1, array_2) -%}\n concat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720957, "supported_languages": null}, "macro.dbt_spark.spark__bool_or": {"name": "spark__bool_or", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt_spark.spark__bool_or", "macro_sql": "{% macro spark__bool_or(expression) -%}\n\n max({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.721088, "supported_languages": null}, "macro.dbt_spark.spark__split_part": {"name": "spark__split_part", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_spark.spark__split_part", "macro_sql": "{% macro spark__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% if part_number >= 0 %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[({{ part_number - 1 if part_number > 0 else part_number }})]\n\n {% endset %}\n\n {% else %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[(\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1 + {{ part_number }}\n )]\n\n {% endset %}\n\n {% endif %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722142, "supported_languages": null}, "macro.dbt_spark.spark__array_construct": {"name": "spark__array_construct", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt_spark.spark__array_construct", "macro_sql": "{% macro spark__array_construct(inputs, data_type) -%}\n array( {{ inputs|join(' , ') }} )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722311, "supported_languages": null}, "macro.dbt_spark.spark__array_append": {"name": "spark__array_append", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt_spark.spark__array_append", "macro_sql": "{% macro spark__array_append(array, new_element) -%}\n {{ array_concat(array, array_construct([new_element])) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.array_concat", "macro.dbt.array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722492, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72342, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723584, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723713, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723829, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723947, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724263, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72451, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724753, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7252579, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.725742, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7284422, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7285912, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.728775, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7294762, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7296178, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.729759, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7308888, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7321742, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.73771, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.737952, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738092, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738164, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738283, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7383811, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7385468, "supported_languages": null}, "macro.dbt.get_snapshot_table_column_names": {"name": "get_snapshot_table_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_snapshot_table_column_names", "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at'}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7387378, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where {{ columns.dbt_valid_to }} is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.739788, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7399502, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7403069, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.740653, "supported_languages": null}, "macro.dbt.get_updated_at_column_data_type": {"name": "get_updated_at_column_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_updated_at_column_data_type", "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741133, "supported_languages": null}, "macro.dbt.check_time_data_types": {"name": "check_time_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.check_time_data_types", "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_updated_at_column_data_type", "macro.dbt.get_snapshot_get_time_data_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741546, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.valid_snapshot_target(target_relation, columns) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.746002, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7482162, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749175, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7494242, "supported_languages": null}, "macro.dbt.get_unit_test_sql": {"name": "get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_unit_test_sql", "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_unit_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749598, "supported_languages": null}, "macro.dbt.default__get_unit_test_sql": {"name": "default__get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_unit_test_sql", "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7500942, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7504158, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7507288, "supported_languages": null}, "macro.dbt.materialization_unit_default": {"name": "materialization_unit_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/unit.sql", "original_file_path": "macros/materializations/tests/unit.sql", "unique_id": "macro.dbt.materialization_unit_default", "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_empty_subquery_sql", "macro.dbt.get_expected_sql", "macro.dbt.get_unit_test_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7521741, "supported_languages": ["sql"]}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756201, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756525, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756736, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.757851, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.75805, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7585862, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.760961, "supported_languages": ["sql"]}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.763325, "supported_languages": ["sql"]}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7646668, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765106, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.76564, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765835, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7664142, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.771425, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.772757, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.77298, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.773788, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774013, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774538, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775049, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775977, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776169, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776327, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7765691, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776725, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7769668, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7771251, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777345, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777499, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777623, "supported_languages": null}, "macro.dbt.get_incremental_microbatch_sql": {"name": "get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_microbatch_sql", "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_microbatch_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777774, "supported_languages": null}, "macro.dbt.default__get_incremental_microbatch_sql": {"name": "default__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_microbatch_sql", "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7779121, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7781398, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.782465, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7869802, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.787961, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.788921, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789614, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789815, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7899132, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790163, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790299, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.793341, "supported_languages": ["sql"]}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.796114, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80035, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80112, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8013299, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801737, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801903, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80202, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8021429, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802243, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8023849, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802489, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8028958, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.803055, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8041549, "supported_languages": null}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804568, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804885, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8053389, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8055522, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8057861, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806116, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806323, "supported_languages": null}, "macro.dbt.get_drop_sql": {"name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806918, "supported_languages": null}, "macro.dbt.default__get_drop_sql": {"name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807232, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8073862, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807553, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807711, "supported_languages": null}, "macro.dbt.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.808418, "supported_languages": null}, "macro.dbt.default__get_replace_sql": {"name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80947, "supported_languages": null}, "macro.dbt.get_create_intermediate_sql": {"name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.809802, "supported_languages": null}, "macro.dbt.default__get_create_intermediate_sql": {"name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810016, "supported_languages": null}, "macro.dbt.drop_schema_named": {"name": "drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.drop_schema_named", "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_schema_named"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81025, "supported_languages": null}, "macro.dbt.default__drop_schema_named": {"name": "default__drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.default__drop_schema_named", "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810434, "supported_languages": null}, "macro.dbt.get_drop_backup_sql": {"name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81071, "supported_languages": null}, "macro.dbt.default__get_drop_backup_sql": {"name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810894, "supported_languages": null}, "macro.dbt.get_rename_sql": {"name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811506, "supported_languages": null}, "macro.dbt.default__get_rename_sql": {"name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811878, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812062, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812324, "supported_languages": null}, "macro.dbt.get_create_backup_sql": {"name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812639, "supported_languages": null}, "macro.dbt.default__get_create_backup_sql": {"name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812882, "supported_languages": null}, "macro.dbt.get_create_sql": {"name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8133051, "supported_languages": null}, "macro.dbt.default__get_create_sql": {"name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813689, "supported_languages": null}, "macro.dbt.get_rename_intermediate_sql": {"name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813972, "supported_languages": null}, "macro.dbt.default__get_rename_intermediate_sql": {"name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8141482, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8143559, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814456, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_sql": {"name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8146849, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_sql": {"name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814875, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815148, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815263, "supported_languages": null}, "macro.dbt.get_rename_materialized_view_sql": {"name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8154979, "supported_languages": null}, "macro.dbt.default__get_rename_materialized_view_sql": {"name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8156219, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8161569, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816313, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8165581, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816685, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816923, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817044, "supported_languages": null}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817939, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818042, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8184981, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818639, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818753, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.819914, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820245, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8205361, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820756, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82086, "supported_languages": null}, "macro.dbt.get_replace_table_sql": {"name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8210938, "supported_languages": null}, "macro.dbt.default__get_replace_table_sql": {"name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8212202, "supported_languages": null}, "macro.dbt.get_rename_table_sql": {"name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82146, "supported_languages": null}, "macro.dbt.default__get_rename_table_sql": {"name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82159, "supported_languages": null}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.822352, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8225112, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8228748, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823435, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8238149, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823968, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824113, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824321, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824422, "supported_languages": null}, "macro.dbt.get_replace_view_sql": {"name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.825161, "supported_languages": null}, "macro.dbt.default__get_replace_view_sql": {"name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82529, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82631, "supported_languages": null}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8264818, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8266811, "supported_languages": null}, "macro.dbt.get_rename_view_sql": {"name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.826918, "supported_languages": null}, "macro.dbt.default__get_rename_view_sql": {"name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8270478, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8274088, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8275461, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.827704, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828066, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828371, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828622, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828834, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.829317, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.83059, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8313391, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8315961, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.833154, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.834102, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8347142, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8349109, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835105, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835169, "supported_languages": null}, "macro.dbt.get_intervals_between": {"name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835808, "supported_languages": null}, "macro.dbt.default__get_intervals_between": {"name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8363059, "supported_languages": null}, "macro.dbt.date_spine": {"name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836503, "supported_languages": null}, "macro.dbt.default__date_spine": {"name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836822, "supported_languages": null}, "macro.dbt.date": {"name": "date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.date", "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837112, "supported_languages": null}, "macro.dbt.default__date": {"name": "default__date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.default__date", "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837353, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837631, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837771, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837983, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838089, "supported_languages": null}, "macro.dbt.get_powers_of_two": {"name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838809, "supported_languages": null}, "macro.dbt.default__get_powers_of_two": {"name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839253, "supported_languages": null}, "macro.dbt.generate_series": {"name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839426, "supported_languages": null}, "macro.dbt.default__generate_series": {"name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839862, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840083, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8401742, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8404648, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840607, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840808, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840875, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841106, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841234, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84149, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8416152, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842151, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842484, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842758, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842896, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843133, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843246, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843461, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843594, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8438652, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8439958, "supported_languages": null}, "macro.dbt.cast": {"name": "cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.cast", "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844224, "supported_languages": null}, "macro.dbt.default__cast": {"name": "default__cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.default__cast", "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844336, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8445451, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844633, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8448782, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844993, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845202, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845294, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846126, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846256, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846394, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8465211, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846658, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8467891, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846924, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8470762, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847214, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847407, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8475451, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8476732, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8478148, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8479412, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848181, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848293, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485062, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485959, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848897, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84913, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.849261, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8497112, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8498502, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8500328, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850267, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.85038, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850707, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850914, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8511622, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851361, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851688, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851847, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851985, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852136, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852708, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852832, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852951, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853037, "supported_languages": null}, "macro.dbt.get_snapshot_get_time_data_type": {"name": "get_snapshot_get_time_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.get_snapshot_get_time_data_type", "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.default__snapshot_get_time", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8533702, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853508, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853577, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853718, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853857, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854583, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854707, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8548422, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8552508, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855412, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855531, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8556669, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855777, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857485, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857631, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857822, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858082, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858318, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8586, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858756, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858906, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859114, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859583, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859786, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859914, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.860286, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86064, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8608792, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8611462, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862612, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862713, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8628511, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862943, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863224, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863383, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863481, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863672, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863852, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86405, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.864309, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8645408, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865141, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865317, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865537, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865737, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8668082, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8671901, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_limit_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8673549, "supported_languages": null}, "macro.dbt.default__get_limit_sql": {"name": "default__get_limit_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_sql", "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ compiled_code }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86752, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86812, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868274, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868457, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8686101, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86885, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8692548, "supported_languages": null}, "macro.dbt.get_catalog_relations": {"name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872187, "supported_languages": null}, "macro.dbt.default__get_catalog_relations": {"name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872406, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872573, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8727858, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872941, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8730712, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8732271, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87344, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8736901, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8739462, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874113, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874256, "supported_languages": null}, "macro.dbt.get_catalog_for_single_relation": {"name": "get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_for_single_relation", "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_catalog_for_single_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874414, "supported_languages": null}, "macro.dbt.default__get_catalog_for_single_relation": {"name": "default__get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_for_single_relation", "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8745549, "supported_languages": null}, "macro.dbt.get_relations": {"name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874696, "supported_languages": null}, "macro.dbt.default__get_relations": {"name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8748312, "supported_languages": null}, "macro.dbt.get_relation_last_modified": {"name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_relation_last_modified"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87501, "supported_languages": null}, "macro.dbt.default__get_relation_last_modified": {"name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8751612, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877135, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877271, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877537, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8777258, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877902, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8780658, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879136, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879442, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879605, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879909, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880113, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880679, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8809001, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.88154, "supported_languages": null}, "macro.dbt.get_fixture_sql": {"name": "get_fixture_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_fixture_sql", "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- for column in columns_in_relation -%}\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.safe_cast", "macro.dbt.validate_fixture_rows", "macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.884523, "supported_languages": null}, "macro.dbt.get_expected_sql": {"name": "get_expected_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_expected_sql", "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8849862, "supported_languages": null}, "macro.dbt.format_row": {"name": "format_row", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.format_row", "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.string_literal", "macro.dbt.escape_single_quotes", "macro.dbt.safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8859332, "supported_languages": null}, "macro.dbt.validate_fixture_rows": {"name": "validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.validate_fixture_rows", "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__validate_fixture_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8861172, "supported_languages": null}, "macro.dbt.default__validate_fixture_rows": {"name": "default__validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.default__validate_fixture_rows", "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.886199, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887635, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887774, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.888478, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8888261, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8893209, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8898182, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.889886, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890347, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8905559, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890809, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.891062, "supported_languages": null}}, "docs": {"doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {}, "groups": {}, "selectors": {}, "disabled": {}, "parent_map": {"model.dbt_demo.diamonds_list_colors": ["model.dbt_demo.diamonds_four_cs"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["source.dbt_demo.raw.diamonds"], "model.dbt_demo.zzz_game_details": [], "model.dbt_demo.second_dbt_model": ["model.dbt_demo.first_dbt_model"], "model.dbt_demo.first_dbt_model": [], "test.dbt_demo.zzz_game_details_check_scores": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_game_details_check_dates": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_win_loss_records_check_records": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.diamonds_list_colors_test": ["model.dbt_demo.diamonds_list_colors"], "seed.dbt_demo.country_codes": [], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": ["model.dbt_demo.second_dbt_model"], "model.dbt_demo.changed_country": ["snapshot.dbt_demo.country_snapshot"], "model.dbt_demo.country": ["seed.dbt_demo.country_codes"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.country"], "model.dbt_demo.zzz_win_loss_records": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": ["model.dbt_demo.zzz_win_loss_records"], "source.dbt_demo.raw.diamonds": [], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": ["model.dbt_demo.diamonds_list_colors"]}, "child_map": {"model.dbt_demo.diamonds_list_colors": ["test.dbt_demo.diamonds_list_colors_test", "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["model.dbt_demo.diamonds_list_colors"], "model.dbt_demo.zzz_game_details": ["model.dbt_demo.zzz_win_loss_records", "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "test.dbt_demo.zzz_game_details_check_dates", "test.dbt_demo.zzz_game_details_check_scores"], "model.dbt_demo.second_dbt_model": ["test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4"], "model.dbt_demo.first_dbt_model": ["model.dbt_demo.second_dbt_model", "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "test.dbt_demo.unique_first_dbt_model_id.9dda05743e"], "test.dbt_demo.zzz_game_details_check_scores": [], "test.dbt_demo.zzz_game_details_check_dates": [], "test.dbt_demo.zzz_win_loss_records_check_records": [], "test.dbt_demo.diamonds_list_colors_test": [], "seed.dbt_demo.country_codes": ["model.dbt_demo.country"], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": [], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": [], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": [], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": [], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": [], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": [], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": [], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": [], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": [], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": [], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": [], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": [], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": [], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": [], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": [], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": [], "model.dbt_demo.changed_country": [], "model.dbt_demo.country": ["snapshot.dbt_demo.country_snapshot"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.changed_country"], "model.dbt_demo.zzz_win_loss_records": ["test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "test.dbt_demo.zzz_win_loss_records_check_records"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": [], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": [], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": [], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": [], "source.dbt_demo.raw.diamonds": ["model.dbt_demo.diamonds_four_cs"], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": []}, "group_map": {}, "saved_queries": {}, "semantic_models": {}, "unit_tests": {"unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": {"model": "diamonds_list_colors", "given": [{"input": "ref('diamonds_four_cs')", "rows": [{"carat": 0.23, "cut": "Ideal", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "K", "clarity": "SI2"}], "format": "dict", "fixture": null}], "expect": {"rows": [{"color": "E"}, {"color": "K"}], "format": "dict", "fixture": null}, "name": "test_get_distinct_colors", "resource_type": "unit_test", "package_name": "dbt_demo", "path": "sql_model1/test_get_distinct_colors.yml", "original_file_path": "src/models/sql_model1/test_get_distinct_colors.yml", "unique_id": "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors", "fqn": ["dbt_demo", "models", "sql_model1", "diamonds_list_colors", "test_get_distinct_colors"], "description": "Check a distinct list of colors is returned.", "overrides": null, "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "config": {"tags": [], "meta": {}}, "checksum": "483c13eb849bf29c60173f6b4583def3c4a3b8c20e674a28c0db2c3e81834bb1", "schema": "dbt_dev", "created_at": 1733486794.4994261, "versions": null, "version": null}}} \ No newline at end of file diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_factory.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_factory.py new file mode 100644 index 00000000..89d10a33 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_factory.py @@ -0,0 +1,461 @@ +import json +from pathlib import Path + +import pytest + +from databricks_dbt_factory.utils import read_dbt_manifest + +BASE_PATH = str(Path(__file__).resolve().parent) + + +def _model(package: str, name: str, depends_on: list[str] | None = None) -> tuple[str, dict]: + full_name = f"model.{package}.{name}" + return full_name, { + "resource_type": "model", + "name": name, + "package_name": package, + "fqn": [package, name], + "depends_on": {"nodes": depends_on or []}, + } + + +def _test( + package: str, + name: str, + depends_on: list[str], + severity: str = "error", + test_hash: str = "", +) -> tuple[str, dict]: + full_name = f"test.{package}.{name}" + (f".{test_hash}" if test_hash else "") + return full_name, { + "resource_type": "test", + "name": name, + "package_name": package, + "fqn": [package, name], + "depends_on": {"nodes": depends_on}, + "config": {"severity": severity}, + } + + +def _seed(package: str, name: str) -> tuple[str, dict]: + full_name = f"seed.{package}.{name}" + return full_name, { + "resource_type": "seed", + "name": name, + "package_name": package, + "fqn": [package, name], + "depends_on": {"nodes": []}, + } + + +def _snapshot(package: str, name: str, depends_on: list[str] | None = None) -> tuple[str, dict]: + full_name = f"snapshot.{package}.{name}" + return full_name, { + "resource_type": "snapshot", + "name": name, + "package_name": package, + "fqn": [package, name], + "depends_on": {"nodes": depends_on or []}, + } + + +def _source(package: str, source_name: str, table: str) -> tuple[str, dict]: + full_name = f"source.{package}.{source_name}.{table}" + return full_name, { + "resource_type": "source", + "name": table, + "source_name": source_name, + "package_name": package, + "fqn": [package, source_name, table], + } + + +def _commands(task: dict) -> list[str]: + """Extracts the dbt commands from a rendered (serverless notebook) task dict.""" + return json.loads(task["notebook_task"]["base_parameters"]["dbt_commands"]) + + +def test_bundled_model_gates_on_tests_of_all_tested_upstreams(dbt_factory_bundled): + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "products"), + _model( + "pkg", + "orders", + depends_on=["model.pkg.customers", "model.pkg.products"], + ), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test("pkg", "unique_products_id", ["model.pkg.products"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "customers_test" in by_key + assert "products_test" in by_key + assert _commands(by_key["customers_test"]) == [ + "dbt test --select fqn:pkg.unique_customers_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["customers_test"]["depends_on"] == [{"task_key": "customers_model"}] + + # orders depends on both upstreams' bundled test tasks (rewired from their run tasks) + assert {dep["task_key"] for dep in by_key["orders_model"]["depends_on"]} == { + "customers_test", + "products_test", + } + + +def test_tests_on_seed_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _seed("pkg", "countries"), + _model("pkg", "enriched", depends_on=["seed.pkg.countries"]), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "countries_test" in by_key + assert _commands(by_key["countries_test"]) == [ + "dbt test --select fqn:pkg.unique_countries_code,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["countries_test"]["depends_on"] == [{"task_key": "countries_seed"}] + assert by_key["enriched_model"]["depends_on"] == [{"task_key": "countries_test"}] + + +def test_tests_on_snapshot_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _snapshot("pkg", "orders_snap"), + _model("pkg", "orders_history", depends_on=["snapshot.pkg.orders_snap"]), + _test("pkg", "not_null_orders_snap_id", ["snapshot.pkg.orders_snap"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "orders_snap_test" in by_key + assert _commands(by_key["orders_snap_test"]) == [ + "dbt test --select fqn:pkg.not_null_orders_snap_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["orders_snap_test"]["depends_on"] == [{"task_key": "orders_snap_snapshot"}] + assert by_key["orders_history_model"]["depends_on"] == [{"task_key": "orders_snap_test"}] + + +def test_tests_on_source_produce_standalone_task(dbt_factory_bundled): + nodes = dict( + [ + _test("pkg", "unique_raw_customers_id", ["source.pkg.raw.customers"]), + ] + ) + sources = dict([_source("pkg", "raw", "customers")]) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes, "sources": sources}) + by_key = {t["task_key"]: t for t in tasks} + + assert "raw_customers_test" in by_key + assert _commands(by_key["raw_customers_test"]) == [ + "dbt test --select fqn:pkg.unique_raw_customers_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["raw_customers_test"]["depends_on"] == [] + + +def test_flat_mode_emits_one_task_per_test_node_and_gates_downstream(dbt_factory): + # Per-test mode mirrors `dbt build`: downstream models wait on upstream tests, so a + # failing `severity: error` test skips downstream via Databricks task failure. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test("pkg", "not_null_customers_id", ["model.pkg.customers"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "unique_customers_id_test" in by_key + assert "not_null_customers_id_test" in by_key + assert "customers_test" not in by_key # no bundling in per-test mode + + assert _commands(by_key["unique_customers_id_test"]) == [ + "dbt test --select fqn:pkg.unique_customers_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["unique_customers_id_test"]["depends_on"] == [{"task_key": "customers_model"}] + # orders depends on customers AND every test attached to customers + assert {dep["task_key"] for dep in by_key["orders_model"]["depends_on"]} == { + "customers_model", + "unique_customers_id_test", + "not_null_customers_id_test", + } + + +def test_flat_mode_cross_model_test_does_not_create_cycle(dbt_factory): + # Relationship test references BOTH `orders` and `customers`. Without care, extending + # `orders`'s deps with "tests of upstream (customers)" would pull in the relationship test, + # which itself depends on `orders` — a direct cycle. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _model("pkg", "payments", depends_on=["model.pkg.orders"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test( + "pkg", + "relationships_orders_customer_id__ref_customers", + ["model.pkg.orders", "model.pkg.customers"], + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # orders depends on customers + unique_customers_id, but NOT on the relationship test + # (that test references orders itself — including it would cycle) + assert {dep["task_key"] for dep in by_key["orders_model"]["depends_on"]} == { + "customers_model", + "unique_customers_id_test", + } + + # payments (downstream of orders) picks up the relationship test — safe, payments + # transitively depends on both orders and customers (the test's refs) + payments_deps = {dep["task_key"] for dep in by_key["payments_model"]["depends_on"]} + assert "orders_model" in payments_deps + assert "relationships_orders_customer_id__ref_customers_test" in payments_deps + + +def test_flat_mode_transitive_cross_model_test_does_not_create_cycle(dbt_factory): + # Transitive cycle case: test T refs {A, C} where C is downstream of B which is downstream + # of A. Extending B's deps with "tests of upstream (A)" must NOT add T, because T depends + # on C and C depends on B → B → T → C → B cycle. Only nodes downstream of both A and C + # (i.e. downstream of C) should get T. + nodes = dict( + [ + _model("pkg", "a"), + _model("pkg", "b", depends_on=["model.pkg.a"]), + _model("pkg", "c", depends_on=["model.pkg.b"]), + _model("pkg", "d", depends_on=["model.pkg.c"]), + _test("pkg", "relationship_a_c", ["model.pkg.a", "model.pkg.c"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # B's ancestors = {A}. Test T refs = {A, C}. C ∉ ancestors(B) → skip T. + assert by_key["b_model"]["depends_on"] == [{"task_key": "a_model"}] + # C's ancestors = {A, B}. C IS in T.refs → skip T (direct self-reference). + assert by_key["c_model"]["depends_on"] == [{"task_key": "b_model"}] + # D's ancestors = {A, B, C}. T.refs = {A, C} ⊆ ancestors(D) → add T. + d_deps = {dep["task_key"] for dep in by_key["d_model"]["depends_on"]} + assert d_deps == {"c_model", "relationship_a_c_test"} + + +def test_flat_mode_tests_gate_downstream_regardless_of_severity(dbt_factory): + # Gating follows dbt's dependency graph, not a test's severity: a warn test can still fail the + # run under `--warn-error`, so both warn- and error-severity tests gate their downstream models. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"], severity="warn"), + _test( + "pkg", + "not_null_customers_id", + ["model.pkg.customers"], + severity="error", + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "unique_customers_id_test" in by_key + assert "not_null_customers_id_test" in by_key + + # orders gates on customers and both attached tests, warn and error alike + assert {dep["task_key"] for dep in by_key["orders_model"]["depends_on"]} == { + "customers_model", + "unique_customers_id_test", + "not_null_customers_id_test", + } + + +def test_flat_mode_test_on_seed_gates_on_seed(dbt_factory): + nodes = dict( + [ + _seed("pkg", "countries"), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert by_key["unique_countries_code_test"]["depends_on"] == [{"task_key": "countries_seed"}] + + +def test_bundled_task_factory_assembles_commands(dbt_factory_bundled): + test_factory = dbt_factory_bundled.task_factories["test"] + task = test_factory.create_bundled_task( + task_key="customers_test", + selects_by_indirect_selection={"cautious": ["fqn:pkg.unique_customers_id,package:pkg,resource_type:test"]}, + deps_command_name="customers", + depends_on=["customers_model"], + ) + assert task.task_key == "customers_test" + assert task.commands == [ + "dbt test --select fqn:pkg.unique_customers_id,package:pkg,resource_type:test --target dev --indirect-selection cautious" + ] + assert task.depends_on == ["customers_model"] + + +def test_cross_model_test_in_bundled_mode_is_emitted_as_standalone_task( + dbt_factory_bundled, +): + # The relationship test spans two models, so it must NOT be collapsed into either model's + # bundled test task (dbt would hit a TABLE_OR_VIEW_NOT_FOUND on the un-built endpoint). + # It should emit its own task with deps on both referenced models. + nodes = dict( + [ + _model("pkg", "team_cities"), + _model("pkg", "game_details", depends_on=["model.pkg.team_cities"]), + _test("pkg", "not_null_team_cities_name", ["model.pkg.team_cities"]), + _test( + "pkg", + "relationships_game_details_winner__team_city__ref_team_cities_", + ["model.pkg.game_details", "model.pkg.team_cities"], + ), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # Single-model test → its own bundled task addressing the exact test node + assert "team_cities_test" in by_key + assert _commands(by_key["team_cities_test"]) == [ + "dbt test --select fqn:pkg.not_null_team_cities_name,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + + # Cross-model test → its own task, gated on BOTH referenced models + cross_test_key = "relationships_game_details_winner__team_city__ref_team_cities__test" + assert cross_test_key in by_key + assert _commands(by_key[cross_test_key]) == [ + "dbt test --select fqn:pkg.relationships_game_details_winner__team_city__ref_team_cities_,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert {dep["task_key"] for dep in by_key[cross_test_key]["depends_on"]} == { + "team_cities_model", + "game_details_model", + } + + # `game_details` has no single-model tests, so no bundled `game_details_test` exists + assert "game_details_test" not in by_key + + +def test_single_package_bundled_test_uses_qualified_select(dbt_factory_bundled): + nodes = dict( + [ + _model("pkg_a", "customers"), + _model("pkg_a", "orders", depends_on=["model.pkg_a.customers"]), + _test("pkg_a", "unique_customers_id", ["model.pkg_a.customers"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "customers_test" in by_key + assert _commands(by_key["customers_test"]) == [ + "dbt test --select fqn:pkg_a.unique_customers_id,package:pkg_a,resource_type:test --target dev --indirect-selection empty" + ] + assert by_key["orders_model"]["depends_on"] == [{"task_key": "customers_test"}] + + +def test_flat_mode_two_tests_sharing_a_name_and_fqn_are_refused(dbt_factory): + # dbt lets the same custom test name land twice with the same fqn, telling the nodes apart only + # via the unique_id hash — which dbt exposes as no selector method. The factory cannot prove a + # selector addresses one without also running the other before its own dependencies are ready, + # so it refuses rather than emit a task that runs the wrong node. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "dup_check", ["model.pkg.customers"], test_hash="6ea6b2ac82"), + _test("pkg", "dup_check", ["model.pkg.customers"], test_hash="a9ab3a6e12"), + ] + ) + + with pytest.raises(ValueError, match="also runs"): + dbt_factory.create_tasks({"nodes": nodes}) + + +def test_flat_mode_cross_package_models_get_package_prefixed_keys(dbt_factory): + # Two packages may ship a model with the same name (materialized to different schemas). + nodes = dict( + [ + _model("shop", "stg_orders"), + _model("subpkg", "stg_orders"), + _model( + "shop", + "mart", + depends_on=["model.shop.stg_orders", "model.subpkg.stg_orders"], + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "shop_stg_orders_model" in by_key + assert "subpkg_stg_orders_model" in by_key + assert {dep["task_key"] for dep in by_key["mart_model"]["depends_on"]} == { + "shop_stg_orders_model", + "subpkg_stg_orders_model", + } + + +def test_bundled_mode_singular_test_named_like_tested_model_keeps_keys_unique( + dbt_factory_bundled, +): + # The bundled test task for model `orders` claims `orders_test`, and so does a singular + # test file named `orders`. Both stay unique (and deploy cannot fail on a duplicate key). + nodes = dict( + [ + _model("pkg", "orders"), + _test("pkg", "unique_orders_id", ["model.pkg.orders"], test_hash="9a1b2c3d4e"), + _test("pkg", "orders", []), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + keys = [t["task_key"] for t in tasks] + by_key = {t["task_key"]: t for t in tasks} + + assert len(keys) == len(set(keys)) + assert _commands(by_key["pkg_orders_test"]) == [ + "dbt test --select fqn:pkg.unique_orders_id,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + assert _commands(by_key["pkg_orders_test_2"]) == [ + "dbt test --select fqn:pkg.orders,package:pkg,resource_type:test --target dev --indirect-selection empty" + ] + + +def test_generated_tasks_match_expected(dbt_factory): + """Safety check: the full set of tasks generated for the sample manifest matches the saved + copy in test_data/expected_tasks.json. If you change the generated output on purpose, refresh + that file with `make test-update-expected-tasks`. + """ + manifest = read_dbt_manifest(BASE_PATH + "/test_data/manifest.json") + tasks = dbt_factory.create_tasks(manifest) + + expected = json.loads(Path(BASE_PATH + "/test_data/expected_tasks.json").read_text()) + assert tasks == expected diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_task.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_task.py new file mode 100644 index 00000000..cb7a24ca --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_task.py @@ -0,0 +1,45 @@ +import json + +from databricks_dbt_factory.dbt_task import DbtTask, DbtTaskOptions + + +def test_renders_serverless_notebook_task(): + options = DbtTaskOptions( + environment_key="Default", + notebook_path="./notebooks/dbt_runner.py", + project_directory="/project", + profiles_directory="/profiles", + ) + task = DbtTask( + task_key="my_model", + commands=["dbt run --select my_model --target dev"], + options=options, + depends_on=["upstream_model"], + ) + + result = task.to_dict() + + assert result["task_key"] == "my_model" + assert result["environment_key"] == "Default" # serverless + assert "job_cluster_key" not in result + assert result["depends_on"] == [{"task_key": "upstream_model"}] + + notebook_task = result["notebook_task"] + assert notebook_task["notebook_path"] == "./notebooks/dbt_runner.py" + assert notebook_task["base_parameters"]["project_directory"] == "/project" + assert notebook_task["base_parameters"]["profiles_directory"] == "/profiles" + assert json.loads(notebook_task["base_parameters"]["dbt_commands"]) == ["dbt run --select my_model --target dev"] + + +def test_defaults_and_optional_directories_are_omitted(): + options = DbtTaskOptions(notebook_path="./runner.py") + task = DbtTask(task_key="m", commands=["dbt run --select m"], options=options) + + result = task.to_dict() + + assert result["environment_key"] == "Default" # default serverless environment + assert result["depends_on"] == [] + base_parameters = result["notebook_task"]["base_parameters"] + assert "project_directory" not in base_parameters + assert "profiles_directory" not in base_parameters + assert json.loads(base_parameters["dbt_commands"]) == ["dbt run --select m"] diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_load_resources.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_load_resources.py new file mode 100644 index 00000000..aa102768 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_load_resources.py @@ -0,0 +1,121 @@ +""" +Offline test for the PyDABs integration in resources/__init__.py. + +Exercises the whole glue path against the committed dbt manifest (no Databricks workspace +required): reading the manifest, wiring the factory, generating notebook tasks, and registering +the job on a Resources object (which runs Job.from_dict, validating the task dict schema). +""" + +import json +from importlib.metadata import version +from pathlib import Path + +import pytest +import yaml +from databricks.bundles.core import Bundle + +import resources +from databricks_dbt_factory.utils import generate_task_key + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +# The manifest is generated by `make manifest` (dbt parse). In a freshly initialized project it +# does not exist yet, so skip these tests rather than erroring until the user has run it. +pytestmark = pytest.mark.skipif( + not (PROJECT_ROOT / resources.MANIFEST_PATH).exists(), + reason="dbt manifest not found; run `make manifest` (dbt parse) first", +) + + +@pytest.fixture(autouse=True) +def _chdir_to_project_root(monkeypatch): + # resources.MANIFEST_PATH is relative to the bundle root, as it is at deploy time. + monkeypatch.chdir(PROJECT_ROOT) + + +def test_build_tasks_covers_every_buildable_node(): + # Every model/seed/snapshot node gets its own run task regardless of the BUNDLE_TESTS mode + # (tests may be collapsed into per-resource `_test` tasks when bundling is on). + tasks = resources._build_tasks("dev") + task_keys = {task["task_key"] for task in tasks} + + manifest = json.loads((PROJECT_ROOT / resources.MANIFEST_PATH).read_text()) + buildable = [ + uid + for uid, node in manifest.get("nodes", {}).items() + if node.get("resource_type") in {"model", "seed", "snapshot"} + ] + expected_keys = {generate_task_key(uid) for uid in buildable} + + assert buildable, "committed manifest has no model/seed/snapshot nodes" + assert expected_keys <= task_keys + + +def test_generated_tasks_are_serverless_notebook_tasks(): + tasks = resources._build_tasks("dev") + + for task in tasks: + assert "notebook_task" in task, task + assert task["environment_key"] == resources.ENVIRONMENT_KEY + assert "job_cluster_key" not in task # serverless only + notebook_task = task["notebook_task"] + assert notebook_task["notebook_path"] == resources.RUNNER_NOTEBOOK_PATH + # The dbt target is injected into every command. + assert "--target dev" in notebook_task["base_parameters"]["dbt_commands"] + + +def test_environment_pins_installed_dbt_databricks(): + # Single source of truth: the serverless env pins dbt-databricks to the exact + # version installed in the bundle venv (the one used to generate the manifest). + spec = resources._serverless_environment_spec() + assert spec["dependencies"] == [f"dbt-databricks=={version('dbt-databricks')}"] + + # The generated job points its serverless environment at the synced env file. + job = resources.build_job("dev") + assert job.environments[0].spec.base_environment == resources.SERVERLESS_ENV_PATH + + +def test_load_resources_registers_the_job(): + result = resources.load_resources(Bundle(target="dev")) + try: + assert resources.JOB_NAME in result.jobs + job = result.jobs[resources.JOB_NAME] + assert job.environments[0].environment_key == resources.ENVIRONMENT_KEY + + # load_resources writes the base-environment file to the bundle root for + # the bundle to sync; it pins dbt-databricks to the installed version. + env_file = PROJECT_ROOT / resources.SERVERLESS_ENV_FILE + assert env_file.exists() + content = yaml.safe_load(env_file.read_text()) + pin = f"dbt-databricks=={version('dbt-databricks')}" + assert content["dependencies"] == [pin] + finally: + (PROJECT_ROOT / resources.SERVERLESS_ENV_FILE).unlink(missing_ok=True) + + +def test_dependency_pin_rejects_non_pypi_versions(monkeypatch): + # A local or dev build cannot be pip-installed from PyPI when Databricks builds the + # serverless environment, so the deploy must fail early with a clear message. + for installed in ("1.9.0+custom", "1.13.0.dev0", "not-a-version"): + monkeypatch.setattr(resources, "version", lambda name, v=installed: v) + with pytest.raises(RuntimeError, match="PyPI"): + resources._dbt_databricks_dependency() + + +def test_environment_file_write_is_idempotent(): + env_file = PROJECT_ROOT / resources.SERVERLESS_ENV_FILE + try: + resources._write_serverless_environment_file() + env_file.chmod(0o444) + # Content already matches: no write is attempted, so a read-only file is fine. + resources._write_serverless_environment_file() + env_file.chmod(0o644) + + env_file.write_text("stale: true\n") + resources._write_serverless_environment_file() + pin = f"dbt-databricks=={version('dbt-databricks')}" + assert yaml.safe_load(env_file.read_text())["dependencies"] == [pin] + finally: + if env_file.exists(): + env_file.chmod(0o644) + env_file.unlink() diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_utils.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_utils.py new file mode 100644 index 00000000..8be9e302 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_utils.py @@ -0,0 +1,94 @@ +from databricks_dbt_factory.utils import ( + MAX_TASK_KEY_LENGTH, + build_task_key_maps, + bundled_test_key, + generate_task_key, +) + + +def test_resource_keys_use_name_and_type_suffix(): + assert generate_task_key("model.shop.orders") == "orders_model" + assert generate_task_key("seed.shop.countries") == "countries_seed" + assert generate_task_key("snapshot.shop.orders_snap") == "orders_snap_snapshot" + + +def test_versioned_model_keeps_version(): + assert generate_task_key("model.shop.dim_customers.v2") == "dim_customers_v2_model" + + +def test_test_key_drops_package_and_hash(): + assert generate_task_key("test.shop.unique_orders_id.9a1b2c3d4e") == "unique_orders_id_test" + # singular test (no trailing hash segment) + assert generate_task_key("test.shop.assert_positive_amount") == "assert_positive_amount_test" + + +def test_source_key(): + assert generate_task_key("source.shop.raw.customers") == "raw_customers_test" + + +def test_bundled_test_key(): + assert bundled_test_key("model.shop.orders") == "orders_test" + assert bundled_test_key("seed.shop.countries") == "countries_test" + assert bundled_test_key("source.shop.raw.customers") == "raw_customers_test" + + +def test_long_test_key_is_truncated_and_hash_disambiguated(): + long_name = "accepted_values_" + "x" * 200 + key = generate_task_key(f"test.shop.{long_name}.9a1b2c3d4e") + assert len(key) <= MAX_TASK_KEY_LENGTH + assert key.endswith("_9a1b2c3d4e_test") # hash re-added so truncated names stay unique + + +def test_key_maps_keep_plain_keys_when_unique(): + task_keys, bundled = build_task_key_maps(["model.shop.orders", "test.shop.unique_orders_id.9a1b2c3d4e"]) + assert task_keys == { + "model.shop.orders": "orders_model", + "test.shop.unique_orders_id.9a1b2c3d4e": "unique_orders_id_test", + } + assert bundled == {} + + +def test_key_maps_disambiguate_same_named_tests_with_dbt_hash(): + # The same custom test name on two models: dbt keeps them apart only via the unique_id hash. + task_keys, _ = build_task_key_maps(["test.shop.dup_check.6ea6b2ac82", "test.shop.dup_check.a9ab3a6e12"]) + assert task_keys["test.shop.dup_check.6ea6b2ac82"] == "dup_check_6ea6b2ac82_test" + assert task_keys["test.shop.dup_check.a9ab3a6e12"] == "dup_check_a9ab3a6e12_test" + + +def test_key_maps_disambiguate_singular_vs_custom_named_test(): + # A singular test file and a custom-named generic test can share a name; the singular test + # has no hash, so it gets the package folded in instead. + task_keys, _ = build_task_key_maps(["test.shop.raw_customers", "test.shop.raw_customers.e58cc24de2"]) + assert task_keys["test.shop.raw_customers"] == "shop_raw_customers_test" + assert task_keys["test.shop.raw_customers.e58cc24de2"] == "raw_customers_e58cc24de2_test" + + +def test_key_maps_disambiguate_cross_package_models_with_package_name(): + task_keys, _ = build_task_key_maps(["model.shop.a", "model.subpkg.a", "model.shop.b"]) + assert task_keys["model.shop.a"] == "shop_a_model" + assert task_keys["model.subpkg.a"] == "subpkg_a_model" + assert task_keys["model.shop.b"] == "b_model" + + +def test_key_maps_disambiguate_bundled_test_key_against_task_keys(): + # Bundled mode: model `orders` claims the bundled key `orders_test`, and so does a singular + # test named `orders`. Every returned key must still be unique. + task_keys, bundled = build_task_key_maps( + ["model.shop.orders", "test.shop.orders"], + bundled_test_ids=["model.shop.orders"], + ) + all_keys = list(task_keys.values()) + list(bundled.values()) + assert len(all_keys) == len(set(all_keys)) + assert task_keys["model.shop.orders"] == "orders_model" + assert task_keys["test.shop.orders"] == "shop_orders_test_2" + assert bundled["model.shop.orders"] == "shop_orders_test" + + +def test_key_maps_disambiguated_long_test_keys_keep_hash_within_limit(): + long_name = "accepted_values_" + "y" * 200 + ids = [f"test.shop.{long_name}.6ea6b2ac82", f"test.shop.{long_name}.a9ab3a6e12"] + task_keys, _ = build_task_key_maps(ids) + assert len(set(task_keys.values())) == 2 + for uid, key in task_keys.items(): + assert len(key) <= MAX_TASK_KEY_LENGTH + assert key.endswith(f"_{uid.split('.')[3]}_test") diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/update_expected_tasks.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/update_expected_tasks.py new file mode 100644 index 00000000..95ea91d3 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/update_expected_tasks.py @@ -0,0 +1,30 @@ +"""Refresh tests/test_data/expected_tasks.json — the saved copy of the job tasks the factory +generates for the sample manifest. The test `test_generated_tasks_match_expected` compares the +current output against this file, so run this script (via `make test-update-expected-tasks`) whenever +you intentionally change the generated output. + +It builds the factory the exact same way the tests do (conftest.create_dbt_factory), so the saved +copy can't drift from what the tests expect. +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from conftest import create_dbt_factory # noqa: E402 +from databricks_dbt_factory.utils import read_dbt_manifest # noqa: E402 + +TEST_DATA = Path(__file__).resolve().parent / "test_data" + + +def main() -> None: + manifest = read_dbt_manifest(str(TEST_DATA / "manifest.json")) + tasks = create_dbt_factory().create_tasks(manifest) + (TEST_DATA / "expected_tasks.json").write_text(json.dumps(tasks, indent=2) + "\n") + print(f"Wrote {len(tasks)} tasks to {TEST_DATA / 'expected_tasks.json'}") + + +if __name__ == "__main__": + main()