diff --git a/packages/google-cloud-dataproc/.gitattributes b/packages/google-cloud-dataproc/.gitattributes new file mode 100644 index 000000000000..33739cb74e44 --- /dev/null +++ b/packages/google-cloud-dataproc/.gitattributes @@ -0,0 +1,4 @@ +*.ts text eol=lf +*.js text eol=lf +protos/* linguist-generated +**/api-extractor.json linguist-language=JSON-with-Comments diff --git a/packages/google-cloud-dataproc/.prettierignore b/packages/google-cloud-dataproc/.prettierignore new file mode 100644 index 000000000000..9340ad9b86d3 --- /dev/null +++ b/packages/google-cloud-dataproc/.prettierignore @@ -0,0 +1,6 @@ +**/node_modules +**/coverage +test/fixtures +build/ +docs/ +protos/ diff --git a/packages/google-cloud-dataproc/.prettierrc.js b/packages/google-cloud-dataproc/.prettierrc.js new file mode 100644 index 000000000000..7649ee3c254e --- /dev/null +++ b/packages/google-cloud-dataproc/.prettierrc.js @@ -0,0 +1,22 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + + +module.exports = { + ...require('gts/.prettierrc.json') +} diff --git a/packages/google-cloud-dataproc/CODE_OF_CONDUCT.md b/packages/google-cloud-dataproc/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..2add2547a812 --- /dev/null +++ b/packages/google-cloud-dataproc/CODE_OF_CONDUCT.md @@ -0,0 +1,94 @@ + +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual's behavior may have a +negative impact on the project or its community. + +## Conflict Resolution + +We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct. + +If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe. + +Reports should be directed to *googleapis-stewards@google.com*, the +Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out to the Project Steward, please email +opensource@google.com. + +We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone's safety, we may take action without notice. + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/packages/google-cloud-dataproc/CONTRIBUTING.md b/packages/google-cloud-dataproc/CONTRIBUTING.md new file mode 100644 index 000000000000..a8b18d88f3a5 --- /dev/null +++ b/packages/google-cloud-dataproc/CONTRIBUTING.md @@ -0,0 +1,76 @@ +# How to become a contributor and submit your own code + +**Table of contents** + +* [Contributor License Agreements](#contributor-license-agreements) +* [Contributing a patch](#contributing-a-patch) +* [Running the tests](#running-the-tests) +* [Releasing the library](#releasing-the-library) + +## Contributor License Agreements + +We'd love to accept your sample apps and patches! Before we can take them, we +have to jump a couple of legal hurdles. + +Please fill out either the individual or corporate Contributor License Agreement +(CLA). + + * If you are an individual writing original source code and you're sure you + own the intellectual property, then you'll need to sign an [individual CLA](https://developers.google.com/open-source/cla/individual). + * If you work for a company that wants to allow you to contribute your work, + then you'll need to sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). + +Follow either of the two links above to access the appropriate CLA and +instructions for how to sign and return it. Once we receive it, we'll be able to +accept your pull requests. + +## Contributing A Patch + +1. Submit an issue describing your proposed change to the repo in question. +1. The repo owner will respond to your issue promptly. +1. If your proposed change is accepted, and you haven't already done so, sign a + Contributor License Agreement (see details above). +1. Fork the desired repo, develop and test your code changes. +1. Ensure that your code adheres to the existing style in the code to which + you are contributing. +1. Ensure that your code has an appropriate set of tests which all pass. +1. Title your pull request following [Conventional Commits](https://www.conventionalcommits.org/) styling. +1. Submit a pull request. + +### Before you begin + +1. [Select or create a Cloud Platform project][projects]. +1. [Enable billing for your project][billing]. +1. [Enable the Dataproc API][enable_api]. +1. [Set up authentication with a service account][auth] so you can access the + API from your local workstation. + + +## Running the tests + +1. [Prepare your environment for Node.js setup][setup]. + +1. Install dependencies: + + npm install + +1. Run the tests: + + # Run unit tests. + npm test + + # Run sample integration tests. + npm run samples-test + + # Run all system tests. + npm run system-test + +1. Lint (and maybe fix) any changes: + + npm run fix + +[setup]: https://cloud.google.com/nodejs/docs/setup +[projects]: https://console.cloud.google.com/project +[billing]: https://support.google.com/cloud/answer/6293499#enable-billing +[enable_api]: https://console.cloud.google.com/flows/enableapi?apiid=dataproc.googleapis.com +[auth]: https://cloud.google.com/docs/authentication/getting-started \ No newline at end of file diff --git a/packages/google-cloud-dataproc/LICENSE b/packages/google-cloud-dataproc/LICENSE new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/packages/google-cloud-dataproc/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/batches.proto b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/batches.proto index bc57cb52344b..acacd40851c9 100644 --- a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/batches.proto +++ b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/batches.proto @@ -1,4 +1,4 @@ -// Copyright 2025 Google LLC +// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -34,7 +34,9 @@ option java_package = "com.google.cloud.dataproc.v1"; service BatchController { option (google.api.default_host) = "dataproc.googleapis.com"; option (google.api.oauth_scopes) = - "https://www.googleapis.com/auth/cloud-platform"; + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/dataproc," + "https://www.googleapis.com/auth/dataproc.read-only"; // Creates a batch workload that executes asynchronously. rpc CreateBatch(CreateBatchRequest) returns (google.longrunning.Operation) { @@ -65,8 +67,10 @@ service BatchController { option (google.api.method_signature) = "parent"; } - // Deletes the batch workload resource. If the batch is not in terminal state, - // the delete fails and the response returns `FAILED_PRECONDITION`. + // Deletes the batch workload resource. If the batch is not in a + // `CANCELLED`, `SUCCEEDED` or `FAILED` + // [`State`][google.cloud.dataproc.v1.Batch.State], the delete operation fails + // and the response returns `FAILED_PRECONDITION`. rpc DeleteBatch(DeleteBatchRequest) returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v1/{name=projects/*/locations/*/batches/*}" @@ -96,10 +100,9 @@ message CreateBatchRequest { // Optional. A unique ID used to identify the request. If the service // receives two - // [CreateBatchRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.CreateBatchRequest)s - // with the same request_id, the second request is ignored and the - // Operation that corresponds to the first Batch created and stored - // in the backend is returned. + // `CreateBatchRequests` with the same `request_id`, the second request is + // ignored and the operation that corresponds to the first Batch created and + // stored in the backend is returned. // // Recommendation: Set this value to a // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). @@ -144,10 +147,13 @@ message ListBatchesRequest { // A filter is a logical expression constraining the values of various fields // in each batch resource. Filters are case sensitive, and may contain // multiple clauses combined with logical operators (AND/OR). - // Supported fields are `batch_id`, `batch_uuid`, `state`, and `create_time`. + // Supported fields are `batch_id`, `batch_uuid`, `state`, `create_time`, and + // `labels`. // // e.g. `state = RUNNING and create_time < "2023-01-01T00:00:00Z"` - // filters for batches in state RUNNING that were created before 2023-01-01 + // filters for batches in state RUNNING that were created before 2023-01-01. + // `state = RUNNING and labels.environment=production` filters for batches in + // state in a RUNNING state that have a production environment label. // // See https://google.aip.dev/assets/misc/ebnf-filtering.txt for a detailed // description of the filter syntax and a list of supported comparisons. @@ -164,8 +170,8 @@ message ListBatchesRequest { // A list of batch workloads. message ListBatchesResponse { - // The batches from the specified collection. - repeated Batch batches = 1; + // Output only. The batches from the specified collection. + repeated Batch batches = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // A token, which can be sent as `page_token` to retrieve the next page. // If this field is omitted, there are no subsequent pages. @@ -256,6 +262,10 @@ message Batch { // Optional. SparkSql batch config. SparkSqlBatch spark_sql_batch = 7 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. PySpark notebook batch config. + PySparkNotebookBatch pyspark_notebook_batch = 19 + [(google.api.field_behavior) = OPTIONAL]; } // Output only. Runtime information about batch execution. @@ -406,3 +416,27 @@ message SparkSqlBatch { // Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH. repeated string jar_file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; } + +// A configuration for running a PySpark Notebook batch workload. +message PySparkNotebookBatch { + // Required. The HCFS URI of the notebook file to execute. + string notebook_file_uri = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The parameters to pass to the notebook. + map params = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. HCFS URIs of Python files to pass to the PySpark framework. + repeated string python_file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH. + repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. HCFS URIs of files to be placed in the working directory of + // each executor + repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. HCFS URIs of archives to be extracted into the working directory + // of each executor. Supported file types: + // `.jar`, `.tar`, `.tar.gz`, `.tgz`, and `.zip`. + repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; +} diff --git a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/session_templates.proto b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/session_templates.proto index a5bbeb6f3f1c..efa204d2eadb 100644 --- a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/session_templates.proto +++ b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/session_templates.proto @@ -1,4 +1,4 @@ -// Copyright 2025 Google LLC +// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -34,7 +34,9 @@ option java_package = "com.google.cloud.dataproc.v1"; service SessionTemplateController { option (google.api.default_host) = "dataproc.googleapis.com"; option (google.api.oauth_scopes) = - "https://www.googleapis.com/auth/cloud-platform"; + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/dataproc," + "https://www.googleapis.com/auth/dataproc.read-only"; // Create a session template synchronously. rpc CreateSessionTemplate(CreateSessionTemplateRequest) @@ -168,8 +170,11 @@ message SessionTemplate { pattern: "projects/{project}/locations/{location}/sessionTemplates/{template}" }; - // Required. The resource name of the session template. - string name = 1 [(google.api.field_behavior) = REQUIRED]; + // Required. Identifier. The resource name of the session template. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.field_behavior) = IDENTIFIER + ]; // Optional. Brief description of the template. string description = 9 [(google.api.field_behavior) = OPTIONAL]; @@ -183,7 +188,7 @@ message SessionTemplate { // Optional. Jupyter session config. JupyterConfig jupyter_session = 3 [(google.api.field_behavior) = OPTIONAL]; - // Optional. Spark Connect session config. + // Optional. Spark connect session config. SparkConnectConfig spark_connect_session = 11 [(google.api.field_behavior) = OPTIONAL]; } diff --git a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/sessions.proto b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/sessions.proto index aed66dc8d7c0..0391e2e8d509 100644 --- a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/sessions.proto +++ b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/sessions.proto @@ -1,4 +1,4 @@ -// Copyright 2025 Google LLC +// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,7 +33,9 @@ option java_package = "com.google.cloud.dataproc.v1"; service SessionController { option (google.api.default_host) = "dataproc.googleapis.com"; option (google.api.oauth_scopes) = - "https://www.googleapis.com/auth/cloud-platform"; + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/dataproc," + "https://www.googleapis.com/auth/dataproc.read-only"; // Create an interactive session asynchronously. rpc CreateSession(CreateSessionRequest) @@ -275,8 +277,8 @@ message Session { [(google.api.field_behavior) = OUTPUT_ONLY]; } - // Required. The resource name of the session. - string name = 1 [(google.api.field_behavior) = REQUIRED]; + // Identifier. The resource name of the session. + string name = 1 [(google.api.field_behavior) = IDENTIFIER]; // Output only. A session UUID (Unique Universal Identifier). The service // generates this value when it creates the session. @@ -291,7 +293,7 @@ message Session { // Optional. Jupyter session config. JupyterConfig jupyter_session = 4 [(google.api.field_behavior) = OPTIONAL]; - // Optional. Spark Connect session config. + // Optional. Spark connect session config. SparkConnectConfig spark_connect_session = 17 [(google.api.field_behavior) = OPTIONAL]; } @@ -375,5 +377,5 @@ message JupyterConfig { string display_name = 2 [(google.api.field_behavior) = OPTIONAL]; } -// Spark Connect configuration for an interactive session. +// Spark connect configuration for an interactive session. message SparkConnectConfig {} diff --git a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/shared.proto b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/shared.proto index 33fb9aff3cae..a545f55f170d 100644 --- a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/shared.proto +++ b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/shared.proto @@ -1,4 +1,4 @@ -// Copyright 2025 Google LLC +// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -59,8 +59,8 @@ message RuntimeConfig { AutotuningConfig autotuning_config = 6 [(google.api.field_behavior) = OPTIONAL]; - // Optional. Cohort identifier. Identifies families of the workloads having - // the same shape, e.g. daily ETL jobs. + // Optional. Cohort identifier. Identifies families of the workloads that have + // the same shape, for example, daily ETL jobs. string cohort = 7 [(google.api.field_behavior) = OPTIONAL]; } @@ -92,7 +92,12 @@ message ExecutionConfig { repeated string network_tags = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Cloud KMS key to use for encryption. - string kms_key = 7 [(google.api.field_behavior) = OPTIONAL]; + string kms_key = 7 [ + (google.api.field_behavior) = OPTIONAL, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKey" + } + ]; // Optional. Applies to sessions only. The duration to keep the session alive // while it's idling. Exceeding this threshold causes the session to @@ -139,6 +144,15 @@ message ExecutionConfig { // resources on the project(s). AuthenticationConfig authentication_config = 11 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Associates Resource Manager tags with the workload nodes. + // There is a max limit of 30 tags. + // Keys and values can be either in numeric format, such as + // `tagKeys/{tag_key_id}` and `tagValues/{tag_value_id}`, or in namespaced + // format, such as `{org_id|project_id}/{tag_key_short_name}` and + // `{tag_value_short_name}`. + map resource_manager_tags = 12 + [(google.api.field_behavior) = OPTIONAL]; } // Spark History Server configuration for the workload. @@ -214,13 +228,17 @@ message UsageMetrics { // (https://cloud.google.com/dataproc-serverless/pricing)). int64 shuffle_storage_gb_seconds = 2 [(google.api.field_behavior) = OPTIONAL]; - // Optional. Accelerator usage in (`milliAccelerator` x `seconds`) (see - // [Dataproc Serverless pricing] + // Optional. [DEPRECATED] Accelerator usage in (`milliAccelerator` x + // `seconds`) (see [Dataproc Serverless pricing] // (https://cloud.google.com/dataproc-serverless/pricing)). int64 milli_accelerator_seconds = 3 [(google.api.field_behavior) = OPTIONAL]; - // Optional. Accelerator type being used, if any + // Optional. [DEPRECATED] Accelerator type being used, if any string accelerator_type = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The timestamp of the usage metrics. + google.protobuf.Timestamp update_time = 6 + [(google.api.field_behavior) = OPTIONAL]; } // The usage snapshot represents the resources consumed by a workload at a @@ -417,8 +435,13 @@ message GkeNodePoolConfig { // (https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek) // used to encrypt the boot disk attached to each node in the node pool. // Specify the key using the following format: - // projects/KEY_PROJECT_ID/locations/LOCATION/keyRings/RING_NAME/cryptoKeys/KEY_NAME. - string boot_disk_kms_key = 23 [(google.api.field_behavior) = OPTIONAL]; + // `projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}` + string boot_disk_kms_key = 23 [ + (google.api.field_behavior) = OPTIONAL, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKey" + } + ]; // Optional. Whether the nodes are created as [Spot VM instances] // (https://cloud.google.com/compute/docs/instances/spot). @@ -545,7 +568,8 @@ message RepositoryConfig { // Configuration for PyPi repository message PyPiRepositoryConfig { - // Optional. PyPi repository address + // Optional. The PyPi repository address. **Note: This field is not available + // for batch workloads.** string pypi_repository = 1 [(google.api.field_behavior) = OPTIONAL]; } @@ -588,9 +612,6 @@ enum Component { // The Jupyter Notebook. JUPYTER = 1; - // The Jupyter Kernel Gateway. - JUPYTER_KERNEL_GATEWAY = 22; - // The Pig component. PIG = 21; @@ -611,6 +632,9 @@ enum Component { // The Zookeeper service. ZOOKEEPER = 8; + + // The Jupyter Kernel Gateway. + JUPYTER_KERNEL_GATEWAY = 22; } // Actions in response to failure of a resource associated with a cluster. diff --git a/packages/google-cloud-dataproc/protos/protos.d.ts b/packages/google-cloud-dataproc/protos/protos.d.ts index d143de6926b0..5e5762b709fb 100644 --- a/packages/google-cloud-dataproc/protos/protos.d.ts +++ b/packages/google-cloud-dataproc/protos/protos.d.ts @@ -1914,6 +1914,9 @@ export namespace google { /** Batch sparkSqlBatch */ sparkSqlBatch?: (google.cloud.dataproc.v1.ISparkSqlBatch|null); + /** Batch pysparkNotebookBatch */ + pysparkNotebookBatch?: (google.cloud.dataproc.v1.IPySparkNotebookBatch|null); + /** Batch runtimeInfo */ runtimeInfo?: (google.cloud.dataproc.v1.IRuntimeInfo|null); @@ -1975,6 +1978,9 @@ export namespace google { /** Batch sparkSqlBatch. */ public sparkSqlBatch?: (google.cloud.dataproc.v1.ISparkSqlBatch|null); + /** Batch pysparkNotebookBatch. */ + public pysparkNotebookBatch?: (google.cloud.dataproc.v1.IPySparkNotebookBatch|null); + /** Batch runtimeInfo. */ public runtimeInfo?: (google.cloud.dataproc.v1.IRuntimeInfo|null); @@ -2006,7 +2012,7 @@ export namespace google { public stateHistory: google.cloud.dataproc.v1.Batch.IStateHistory[]; /** Batch batchConfig. */ - public batchConfig?: ("pysparkBatch"|"sparkBatch"|"sparkRBatch"|"sparkSqlBatch"); + public batchConfig?: ("pysparkBatch"|"sparkBatch"|"sparkRBatch"|"sparkSqlBatch"|"pysparkNotebookBatch"); /** * Creates a new Batch instance using the specified properties. @@ -2690,6 +2696,133 @@ export namespace google { public static getTypeUrl(typeUrlPrefix?: string): string; } + /** Properties of a PySparkNotebookBatch. */ + interface IPySparkNotebookBatch { + + /** PySparkNotebookBatch notebookFileUri */ + notebookFileUri?: (string|null); + + /** PySparkNotebookBatch params */ + params?: ({ [k: string]: string }|null); + + /** PySparkNotebookBatch pythonFileUris */ + pythonFileUris?: (string[]|null); + + /** PySparkNotebookBatch jarFileUris */ + jarFileUris?: (string[]|null); + + /** PySparkNotebookBatch fileUris */ + fileUris?: (string[]|null); + + /** PySparkNotebookBatch archiveUris */ + archiveUris?: (string[]|null); + } + + /** Represents a PySparkNotebookBatch. */ + class PySparkNotebookBatch implements IPySparkNotebookBatch { + + /** + * Constructs a new PySparkNotebookBatch. + * @param [properties] Properties to set + */ + constructor(properties?: google.cloud.dataproc.v1.IPySparkNotebookBatch); + + /** PySparkNotebookBatch notebookFileUri. */ + public notebookFileUri: string; + + /** PySparkNotebookBatch params. */ + public params: { [k: string]: string }; + + /** PySparkNotebookBatch pythonFileUris. */ + public pythonFileUris: string[]; + + /** PySparkNotebookBatch jarFileUris. */ + public jarFileUris: string[]; + + /** PySparkNotebookBatch fileUris. */ + public fileUris: string[]; + + /** PySparkNotebookBatch archiveUris. */ + public archiveUris: string[]; + + /** + * Creates a new PySparkNotebookBatch instance using the specified properties. + * @param [properties] Properties to set + * @returns PySparkNotebookBatch instance + */ + public static create(properties?: google.cloud.dataproc.v1.IPySparkNotebookBatch): google.cloud.dataproc.v1.PySparkNotebookBatch; + + /** + * Encodes the specified PySparkNotebookBatch message. Does not implicitly {@link google.cloud.dataproc.v1.PySparkNotebookBatch.verify|verify} messages. + * @param message PySparkNotebookBatch message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encode(message: google.cloud.dataproc.v1.IPySparkNotebookBatch, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Encodes the specified PySparkNotebookBatch message, length delimited. Does not implicitly {@link google.cloud.dataproc.v1.PySparkNotebookBatch.verify|verify} messages. + * @param message PySparkNotebookBatch message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encodeDelimited(message: google.cloud.dataproc.v1.IPySparkNotebookBatch, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Decodes a PySparkNotebookBatch message from the specified reader or buffer. + * @param reader Reader or buffer to decode from + * @param [length] Message length if known beforehand + * @returns PySparkNotebookBatch + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): google.cloud.dataproc.v1.PySparkNotebookBatch; + + /** + * Decodes a PySparkNotebookBatch message from the specified reader or buffer, length delimited. + * @param reader Reader or buffer to decode from + * @returns PySparkNotebookBatch + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decodeDelimited(reader: ($protobuf.Reader|Uint8Array)): google.cloud.dataproc.v1.PySparkNotebookBatch; + + /** + * Verifies a PySparkNotebookBatch message. + * @param message Plain object to verify + * @returns `null` if valid, otherwise the reason why it is not + */ + public static verify(message: { [k: string]: any }): (string|null); + + /** + * Creates a PySparkNotebookBatch message from a plain object. Also converts values to their respective internal types. + * @param object Plain object + * @returns PySparkNotebookBatch + */ + public static fromObject(object: { [k: string]: any }): google.cloud.dataproc.v1.PySparkNotebookBatch; + + /** + * Creates a plain object from a PySparkNotebookBatch message. Also converts values to other types if specified. + * @param message PySparkNotebookBatch + * @param [options] Conversion options + * @returns Plain object + */ + public static toObject(message: google.cloud.dataproc.v1.PySparkNotebookBatch, options?: $protobuf.IConversionOptions): { [k: string]: any }; + + /** + * Converts this PySparkNotebookBatch to JSON. + * @returns JSON object + */ + public toJSON(): { [k: string]: any }; + + /** + * Gets the default type url for PySparkNotebookBatch + * @param [typeUrlPrefix] your custom typeUrlPrefix(default "type.googleapis.com") + * @returns The default type url + */ + public static getTypeUrl(typeUrlPrefix?: string): string; + } + /** Properties of a RuntimeConfig. */ interface IRuntimeConfig { @@ -2949,6 +3082,9 @@ export namespace google { /** ExecutionConfig authenticationConfig */ authenticationConfig?: (google.cloud.dataproc.v1.IAuthenticationConfig|null); + + /** ExecutionConfig resourceManagerTags */ + resourceManagerTags?: ({ [k: string]: string }|null); } /** Represents an ExecutionConfig. */ @@ -2987,6 +3123,9 @@ export namespace google { /** ExecutionConfig authenticationConfig. */ public authenticationConfig?: (google.cloud.dataproc.v1.IAuthenticationConfig|null); + /** ExecutionConfig resourceManagerTags. */ + public resourceManagerTags: { [k: string]: string }; + /** ExecutionConfig network. */ public network?: ("networkUri"|"subnetworkUri"); @@ -3403,6 +3542,9 @@ export namespace google { /** UsageMetrics acceleratorType */ acceleratorType?: (string|null); + + /** UsageMetrics updateTime */ + updateTime?: (google.protobuf.ITimestamp|null); } /** Represents a UsageMetrics. */ @@ -3426,6 +3568,9 @@ export namespace google { /** UsageMetrics acceleratorType. */ public acceleratorType: string; + /** UsageMetrics updateTime. */ + public updateTime?: (google.protobuf.ITimestamp|null); + /** * Creates a new UsageMetrics instance using the specified properties. * @param [properties] Properties to set @@ -4957,14 +5102,14 @@ export namespace google { HUDI = 18, ICEBERG = 19, JUPYTER = 1, - JUPYTER_KERNEL_GATEWAY = 22, PIG = 21, PRESTO = 6, TRINO = 17, RANGER = 12, SOLR = 10, ZEPPELIN = 4, - ZOOKEEPER = 8 + ZOOKEEPER = 8, + JUPYTER_KERNEL_GATEWAY = 22 } /** FailureAction enum. */ diff --git a/packages/google-cloud-dataproc/protos/protos.js b/packages/google-cloud-dataproc/protos/protos.js index 573c43086bca..b395f827a793 100644 --- a/packages/google-cloud-dataproc/protos/protos.js +++ b/packages/google-cloud-dataproc/protos/protos.js @@ -4348,6 +4348,7 @@ * @property {google.cloud.dataproc.v1.ISparkBatch|null} [sparkBatch] Batch sparkBatch * @property {google.cloud.dataproc.v1.ISparkRBatch|null} [sparkRBatch] Batch sparkRBatch * @property {google.cloud.dataproc.v1.ISparkSqlBatch|null} [sparkSqlBatch] Batch sparkSqlBatch + * @property {google.cloud.dataproc.v1.IPySparkNotebookBatch|null} [pysparkNotebookBatch] Batch pysparkNotebookBatch * @property {google.cloud.dataproc.v1.IRuntimeInfo|null} [runtimeInfo] Batch runtimeInfo * @property {google.cloud.dataproc.v1.Batch.State|null} [state] Batch state * @property {string|null} [stateMessage] Batch stateMessage @@ -4433,6 +4434,14 @@ */ Batch.prototype.sparkSqlBatch = null; + /** + * Batch pysparkNotebookBatch. + * @member {google.cloud.dataproc.v1.IPySparkNotebookBatch|null|undefined} pysparkNotebookBatch + * @memberof google.cloud.dataproc.v1.Batch + * @instance + */ + Batch.prototype.pysparkNotebookBatch = null; + /** * Batch runtimeInfo. * @member {google.cloud.dataproc.v1.IRuntimeInfo|null|undefined} runtimeInfo @@ -4518,12 +4527,12 @@ /** * Batch batchConfig. - * @member {"pysparkBatch"|"sparkBatch"|"sparkRBatch"|"sparkSqlBatch"|undefined} batchConfig + * @member {"pysparkBatch"|"sparkBatch"|"sparkRBatch"|"sparkSqlBatch"|"pysparkNotebookBatch"|undefined} batchConfig * @memberof google.cloud.dataproc.v1.Batch * @instance */ Object.defineProperty(Batch.prototype, "batchConfig", { - get: $util.oneOfGetter($oneOfFields = ["pysparkBatch", "sparkBatch", "sparkRBatch", "sparkSqlBatch"]), + get: $util.oneOfGetter($oneOfFields = ["pysparkBatch", "sparkBatch", "sparkRBatch", "sparkSqlBatch", "pysparkNotebookBatch"]), set: $util.oneOfSetter($oneOfFields) }); @@ -4587,6 +4596,8 @@ if (message.stateHistory != null && message.stateHistory.length) for (var i = 0; i < message.stateHistory.length; ++i) $root.google.cloud.dataproc.v1.Batch.StateHistory.encode(message.stateHistory[i], writer.uint32(/* id 17, wireType 2 =*/138).fork()).ldelim(); + if (message.pysparkNotebookBatch != null && Object.hasOwnProperty.call(message, "pysparkNotebookBatch")) + $root.google.cloud.dataproc.v1.PySparkNotebookBatch.encode(message.pysparkNotebookBatch, writer.uint32(/* id 19, wireType 2 =*/154).fork()).ldelim(); return writer; }; @@ -4651,6 +4662,10 @@ message.sparkSqlBatch = $root.google.cloud.dataproc.v1.SparkSqlBatch.decode(reader, reader.uint32()); break; } + case 19: { + message.pysparkNotebookBatch = $root.google.cloud.dataproc.v1.PySparkNotebookBatch.decode(reader, reader.uint32()); + break; + } case 8: { message.runtimeInfo = $root.google.cloud.dataproc.v1.RuntimeInfo.decode(reader, reader.uint32()); break; @@ -4797,6 +4812,16 @@ return "sparkSqlBatch." + error; } } + if (message.pysparkNotebookBatch != null && message.hasOwnProperty("pysparkNotebookBatch")) { + if (properties.batchConfig === 1) + return "batchConfig: multiple values"; + properties.batchConfig = 1; + { + var error = $root.google.cloud.dataproc.v1.PySparkNotebookBatch.verify(message.pysparkNotebookBatch); + if (error) + return "pysparkNotebookBatch." + error; + } + } if (message.runtimeInfo != null && message.hasOwnProperty("runtimeInfo")) { var error = $root.google.cloud.dataproc.v1.RuntimeInfo.verify(message.runtimeInfo); if (error) @@ -4900,6 +4925,11 @@ throw TypeError(".google.cloud.dataproc.v1.Batch.sparkSqlBatch: object expected"); message.sparkSqlBatch = $root.google.cloud.dataproc.v1.SparkSqlBatch.fromObject(object.sparkSqlBatch); } + if (object.pysparkNotebookBatch != null) { + if (typeof object.pysparkNotebookBatch !== "object") + throw TypeError(".google.cloud.dataproc.v1.Batch.pysparkNotebookBatch: object expected"); + message.pysparkNotebookBatch = $root.google.cloud.dataproc.v1.PySparkNotebookBatch.fromObject(object.pysparkNotebookBatch); + } if (object.runtimeInfo != null) { if (typeof object.runtimeInfo !== "object") throw TypeError(".google.cloud.dataproc.v1.Batch.runtimeInfo: object expected"); @@ -5065,6 +5095,11 @@ for (var j = 0; j < message.stateHistory.length; ++j) object.stateHistory[j] = $root.google.cloud.dataproc.v1.Batch.StateHistory.toObject(message.stateHistory[j], options); } + if (message.pysparkNotebookBatch != null && message.hasOwnProperty("pysparkNotebookBatch")) { + object.pysparkNotebookBatch = $root.google.cloud.dataproc.v1.PySparkNotebookBatch.toObject(message.pysparkNotebookBatch, options); + if (options.oneofs) + object.batchConfig = "pysparkNotebookBatch"; + } return object; }; @@ -6862,6 +6897,428 @@ return SparkSqlBatch; })(); + v1.PySparkNotebookBatch = (function() { + + /** + * Properties of a PySparkNotebookBatch. + * @memberof google.cloud.dataproc.v1 + * @interface IPySparkNotebookBatch + * @property {string|null} [notebookFileUri] PySparkNotebookBatch notebookFileUri + * @property {Object.|null} [params] PySparkNotebookBatch params + * @property {Array.|null} [pythonFileUris] PySparkNotebookBatch pythonFileUris + * @property {Array.|null} [jarFileUris] PySparkNotebookBatch jarFileUris + * @property {Array.|null} [fileUris] PySparkNotebookBatch fileUris + * @property {Array.|null} [archiveUris] PySparkNotebookBatch archiveUris + */ + + /** + * Constructs a new PySparkNotebookBatch. + * @memberof google.cloud.dataproc.v1 + * @classdesc Represents a PySparkNotebookBatch. + * @implements IPySparkNotebookBatch + * @constructor + * @param {google.cloud.dataproc.v1.IPySparkNotebookBatch=} [properties] Properties to set + */ + function PySparkNotebookBatch(properties) { + this.params = {}; + this.pythonFileUris = []; + this.jarFileUris = []; + this.fileUris = []; + this.archiveUris = []; + if (properties) + for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) + if (properties[keys[i]] != null) + this[keys[i]] = properties[keys[i]]; + } + + /** + * PySparkNotebookBatch notebookFileUri. + * @member {string} notebookFileUri + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @instance + */ + PySparkNotebookBatch.prototype.notebookFileUri = ""; + + /** + * PySparkNotebookBatch params. + * @member {Object.} params + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @instance + */ + PySparkNotebookBatch.prototype.params = $util.emptyObject; + + /** + * PySparkNotebookBatch pythonFileUris. + * @member {Array.} pythonFileUris + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @instance + */ + PySparkNotebookBatch.prototype.pythonFileUris = $util.emptyArray; + + /** + * PySparkNotebookBatch jarFileUris. + * @member {Array.} jarFileUris + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @instance + */ + PySparkNotebookBatch.prototype.jarFileUris = $util.emptyArray; + + /** + * PySparkNotebookBatch fileUris. + * @member {Array.} fileUris + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @instance + */ + PySparkNotebookBatch.prototype.fileUris = $util.emptyArray; + + /** + * PySparkNotebookBatch archiveUris. + * @member {Array.} archiveUris + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @instance + */ + PySparkNotebookBatch.prototype.archiveUris = $util.emptyArray; + + /** + * Creates a new PySparkNotebookBatch instance using the specified properties. + * @function create + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @static + * @param {google.cloud.dataproc.v1.IPySparkNotebookBatch=} [properties] Properties to set + * @returns {google.cloud.dataproc.v1.PySparkNotebookBatch} PySparkNotebookBatch instance + */ + PySparkNotebookBatch.create = function create(properties) { + return new PySparkNotebookBatch(properties); + }; + + /** + * Encodes the specified PySparkNotebookBatch message. Does not implicitly {@link google.cloud.dataproc.v1.PySparkNotebookBatch.verify|verify} messages. + * @function encode + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @static + * @param {google.cloud.dataproc.v1.IPySparkNotebookBatch} message PySparkNotebookBatch message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + PySparkNotebookBatch.encode = function encode(message, writer) { + if (!writer) + writer = $Writer.create(); + if (message.notebookFileUri != null && Object.hasOwnProperty.call(message, "notebookFileUri")) + writer.uint32(/* id 1, wireType 2 =*/10).string(message.notebookFileUri); + if (message.params != null && Object.hasOwnProperty.call(message, "params")) + for (var keys = Object.keys(message.params), i = 0; i < keys.length; ++i) + writer.uint32(/* id 2, wireType 2 =*/18).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.params[keys[i]]).ldelim(); + if (message.pythonFileUris != null && message.pythonFileUris.length) + for (var i = 0; i < message.pythonFileUris.length; ++i) + writer.uint32(/* id 3, wireType 2 =*/26).string(message.pythonFileUris[i]); + if (message.jarFileUris != null && message.jarFileUris.length) + for (var i = 0; i < message.jarFileUris.length; ++i) + writer.uint32(/* id 4, wireType 2 =*/34).string(message.jarFileUris[i]); + if (message.fileUris != null && message.fileUris.length) + for (var i = 0; i < message.fileUris.length; ++i) + writer.uint32(/* id 5, wireType 2 =*/42).string(message.fileUris[i]); + if (message.archiveUris != null && message.archiveUris.length) + for (var i = 0; i < message.archiveUris.length; ++i) + writer.uint32(/* id 6, wireType 2 =*/50).string(message.archiveUris[i]); + return writer; + }; + + /** + * Encodes the specified PySparkNotebookBatch message, length delimited. Does not implicitly {@link google.cloud.dataproc.v1.PySparkNotebookBatch.verify|verify} messages. + * @function encodeDelimited + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @static + * @param {google.cloud.dataproc.v1.IPySparkNotebookBatch} message PySparkNotebookBatch message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + PySparkNotebookBatch.encodeDelimited = function encodeDelimited(message, writer) { + return this.encode(message, writer).ldelim(); + }; + + /** + * Decodes a PySparkNotebookBatch message from the specified reader or buffer. + * @function decode + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {google.cloud.dataproc.v1.PySparkNotebookBatch} PySparkNotebookBatch + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + PySparkNotebookBatch.decode = function decode(reader, length, error) { + if (!(reader instanceof $Reader)) + reader = $Reader.create(reader); + var end = length === undefined ? reader.len : reader.pos + length, message = new $root.google.cloud.dataproc.v1.PySparkNotebookBatch(), key, value; + while (reader.pos < end) { + var tag = reader.uint32(); + if (tag === error) + break; + switch (tag >>> 3) { + case 1: { + message.notebookFileUri = reader.string(); + break; + } + case 2: { + if (message.params === $util.emptyObject) + message.params = {}; + var end2 = reader.uint32() + reader.pos; + key = ""; + value = ""; + while (reader.pos < end2) { + var tag2 = reader.uint32(); + switch (tag2 >>> 3) { + case 1: + key = reader.string(); + break; + case 2: + value = reader.string(); + break; + default: + reader.skipType(tag2 & 7); + break; + } + } + message.params[key] = value; + break; + } + case 3: { + if (!(message.pythonFileUris && message.pythonFileUris.length)) + message.pythonFileUris = []; + message.pythonFileUris.push(reader.string()); + break; + } + case 4: { + if (!(message.jarFileUris && message.jarFileUris.length)) + message.jarFileUris = []; + message.jarFileUris.push(reader.string()); + break; + } + case 5: { + if (!(message.fileUris && message.fileUris.length)) + message.fileUris = []; + message.fileUris.push(reader.string()); + break; + } + case 6: { + if (!(message.archiveUris && message.archiveUris.length)) + message.archiveUris = []; + message.archiveUris.push(reader.string()); + break; + } + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Decodes a PySparkNotebookBatch message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {google.cloud.dataproc.v1.PySparkNotebookBatch} PySparkNotebookBatch + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + PySparkNotebookBatch.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) + reader = new $Reader(reader); + return this.decode(reader, reader.uint32()); + }; + + /** + * Verifies a PySparkNotebookBatch message. + * @function verify + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + PySparkNotebookBatch.verify = function verify(message) { + if (typeof message !== "object" || message === null) + return "object expected"; + if (message.notebookFileUri != null && message.hasOwnProperty("notebookFileUri")) + if (!$util.isString(message.notebookFileUri)) + return "notebookFileUri: string expected"; + if (message.params != null && message.hasOwnProperty("params")) { + if (!$util.isObject(message.params)) + return "params: object expected"; + var key = Object.keys(message.params); + for (var i = 0; i < key.length; ++i) + if (!$util.isString(message.params[key[i]])) + return "params: string{k:string} expected"; + } + if (message.pythonFileUris != null && message.hasOwnProperty("pythonFileUris")) { + if (!Array.isArray(message.pythonFileUris)) + return "pythonFileUris: array expected"; + for (var i = 0; i < message.pythonFileUris.length; ++i) + if (!$util.isString(message.pythonFileUris[i])) + return "pythonFileUris: string[] expected"; + } + if (message.jarFileUris != null && message.hasOwnProperty("jarFileUris")) { + if (!Array.isArray(message.jarFileUris)) + return "jarFileUris: array expected"; + for (var i = 0; i < message.jarFileUris.length; ++i) + if (!$util.isString(message.jarFileUris[i])) + return "jarFileUris: string[] expected"; + } + if (message.fileUris != null && message.hasOwnProperty("fileUris")) { + if (!Array.isArray(message.fileUris)) + return "fileUris: array expected"; + for (var i = 0; i < message.fileUris.length; ++i) + if (!$util.isString(message.fileUris[i])) + return "fileUris: string[] expected"; + } + if (message.archiveUris != null && message.hasOwnProperty("archiveUris")) { + if (!Array.isArray(message.archiveUris)) + return "archiveUris: array expected"; + for (var i = 0; i < message.archiveUris.length; ++i) + if (!$util.isString(message.archiveUris[i])) + return "archiveUris: string[] expected"; + } + return null; + }; + + /** + * Creates a PySparkNotebookBatch message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @static + * @param {Object.} object Plain object + * @returns {google.cloud.dataproc.v1.PySparkNotebookBatch} PySparkNotebookBatch + */ + PySparkNotebookBatch.fromObject = function fromObject(object) { + if (object instanceof $root.google.cloud.dataproc.v1.PySparkNotebookBatch) + return object; + var message = new $root.google.cloud.dataproc.v1.PySparkNotebookBatch(); + if (object.notebookFileUri != null) + message.notebookFileUri = String(object.notebookFileUri); + if (object.params) { + if (typeof object.params !== "object") + throw TypeError(".google.cloud.dataproc.v1.PySparkNotebookBatch.params: object expected"); + message.params = {}; + for (var keys = Object.keys(object.params), i = 0; i < keys.length; ++i) + message.params[keys[i]] = String(object.params[keys[i]]); + } + if (object.pythonFileUris) { + if (!Array.isArray(object.pythonFileUris)) + throw TypeError(".google.cloud.dataproc.v1.PySparkNotebookBatch.pythonFileUris: array expected"); + message.pythonFileUris = []; + for (var i = 0; i < object.pythonFileUris.length; ++i) + message.pythonFileUris[i] = String(object.pythonFileUris[i]); + } + if (object.jarFileUris) { + if (!Array.isArray(object.jarFileUris)) + throw TypeError(".google.cloud.dataproc.v1.PySparkNotebookBatch.jarFileUris: array expected"); + message.jarFileUris = []; + for (var i = 0; i < object.jarFileUris.length; ++i) + message.jarFileUris[i] = String(object.jarFileUris[i]); + } + if (object.fileUris) { + if (!Array.isArray(object.fileUris)) + throw TypeError(".google.cloud.dataproc.v1.PySparkNotebookBatch.fileUris: array expected"); + message.fileUris = []; + for (var i = 0; i < object.fileUris.length; ++i) + message.fileUris[i] = String(object.fileUris[i]); + } + if (object.archiveUris) { + if (!Array.isArray(object.archiveUris)) + throw TypeError(".google.cloud.dataproc.v1.PySparkNotebookBatch.archiveUris: array expected"); + message.archiveUris = []; + for (var i = 0; i < object.archiveUris.length; ++i) + message.archiveUris[i] = String(object.archiveUris[i]); + } + return message; + }; + + /** + * Creates a plain object from a PySparkNotebookBatch message. Also converts values to other types if specified. + * @function toObject + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @static + * @param {google.cloud.dataproc.v1.PySparkNotebookBatch} message PySparkNotebookBatch + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + PySparkNotebookBatch.toObject = function toObject(message, options) { + if (!options) + options = {}; + var object = {}; + if (options.arrays || options.defaults) { + object.pythonFileUris = []; + object.jarFileUris = []; + object.fileUris = []; + object.archiveUris = []; + } + if (options.objects || options.defaults) + object.params = {}; + if (options.defaults) + object.notebookFileUri = ""; + if (message.notebookFileUri != null && message.hasOwnProperty("notebookFileUri")) + object.notebookFileUri = message.notebookFileUri; + var keys2; + if (message.params && (keys2 = Object.keys(message.params)).length) { + object.params = {}; + for (var j = 0; j < keys2.length; ++j) + object.params[keys2[j]] = message.params[keys2[j]]; + } + if (message.pythonFileUris && message.pythonFileUris.length) { + object.pythonFileUris = []; + for (var j = 0; j < message.pythonFileUris.length; ++j) + object.pythonFileUris[j] = message.pythonFileUris[j]; + } + if (message.jarFileUris && message.jarFileUris.length) { + object.jarFileUris = []; + for (var j = 0; j < message.jarFileUris.length; ++j) + object.jarFileUris[j] = message.jarFileUris[j]; + } + if (message.fileUris && message.fileUris.length) { + object.fileUris = []; + for (var j = 0; j < message.fileUris.length; ++j) + object.fileUris[j] = message.fileUris[j]; + } + if (message.archiveUris && message.archiveUris.length) { + object.archiveUris = []; + for (var j = 0; j < message.archiveUris.length; ++j) + object.archiveUris[j] = message.archiveUris[j]; + } + return object; + }; + + /** + * Converts this PySparkNotebookBatch to JSON. + * @function toJSON + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @instance + * @returns {Object.} JSON object + */ + PySparkNotebookBatch.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions); + }; + + /** + * Gets the default type url for PySparkNotebookBatch + * @function getTypeUrl + * @memberof google.cloud.dataproc.v1.PySparkNotebookBatch + * @static + * @param {string} [typeUrlPrefix] your custom typeUrlPrefix(default "type.googleapis.com") + * @returns {string} The default type url + */ + PySparkNotebookBatch.getTypeUrl = function getTypeUrl(typeUrlPrefix) { + if (typeUrlPrefix === undefined) { + typeUrlPrefix = "type.googleapis.com"; + } + return typeUrlPrefix + "/google.cloud.dataproc.v1.PySparkNotebookBatch"; + }; + + return PySparkNotebookBatch; + })(); + v1.RuntimeConfig = (function() { /** @@ -7483,6 +7940,7 @@ * @property {google.protobuf.IDuration|null} [ttl] ExecutionConfig ttl * @property {string|null} [stagingBucket] ExecutionConfig stagingBucket * @property {google.cloud.dataproc.v1.IAuthenticationConfig|null} [authenticationConfig] ExecutionConfig authenticationConfig + * @property {Object.|null} [resourceManagerTags] ExecutionConfig resourceManagerTags */ /** @@ -7495,6 +7953,7 @@ */ function ExecutionConfig(properties) { this.networkTags = []; + this.resourceManagerTags = {}; if (properties) for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) if (properties[keys[i]] != null) @@ -7573,6 +8032,14 @@ */ ExecutionConfig.prototype.authenticationConfig = null; + /** + * ExecutionConfig resourceManagerTags. + * @member {Object.} resourceManagerTags + * @memberof google.cloud.dataproc.v1.ExecutionConfig + * @instance + */ + ExecutionConfig.prototype.resourceManagerTags = $util.emptyObject; + // OneOf field names bound to virtual getters and setters var $oneOfFields; @@ -7630,6 +8097,9 @@ writer.uint32(/* id 10, wireType 2 =*/82).string(message.stagingBucket); if (message.authenticationConfig != null && Object.hasOwnProperty.call(message, "authenticationConfig")) $root.google.cloud.dataproc.v1.AuthenticationConfig.encode(message.authenticationConfig, writer.uint32(/* id 11, wireType 2 =*/90).fork()).ldelim(); + if (message.resourceManagerTags != null && Object.hasOwnProperty.call(message, "resourceManagerTags")) + for (var keys = Object.keys(message.resourceManagerTags), i = 0; i < keys.length; ++i) + writer.uint32(/* id 12, wireType 2 =*/98).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.resourceManagerTags[keys[i]]).ldelim(); return writer; }; @@ -7660,7 +8130,7 @@ ExecutionConfig.decode = function decode(reader, length, error) { if (!(reader instanceof $Reader)) reader = $Reader.create(reader); - var end = length === undefined ? reader.len : reader.pos + length, message = new $root.google.cloud.dataproc.v1.ExecutionConfig(); + var end = length === undefined ? reader.len : reader.pos + length, message = new $root.google.cloud.dataproc.v1.ExecutionConfig(), key, value; while (reader.pos < end) { var tag = reader.uint32(); if (tag === error) @@ -7704,6 +8174,29 @@ message.authenticationConfig = $root.google.cloud.dataproc.v1.AuthenticationConfig.decode(reader, reader.uint32()); break; } + case 12: { + if (message.resourceManagerTags === $util.emptyObject) + message.resourceManagerTags = {}; + var end2 = reader.uint32() + reader.pos; + key = ""; + value = ""; + while (reader.pos < end2) { + var tag2 = reader.uint32(); + switch (tag2 >>> 3) { + case 1: + key = reader.string(); + break; + case 2: + value = reader.string(); + break; + default: + reader.skipType(tag2 & 7); + break; + } + } + message.resourceManagerTags[key] = value; + break; + } default: reader.skipType(tag & 7); break; @@ -7783,6 +8276,14 @@ if (error) return "authenticationConfig." + error; } + if (message.resourceManagerTags != null && message.hasOwnProperty("resourceManagerTags")) { + if (!$util.isObject(message.resourceManagerTags)) + return "resourceManagerTags: object expected"; + var key = Object.keys(message.resourceManagerTags); + for (var i = 0; i < key.length; ++i) + if (!$util.isString(message.resourceManagerTags[key[i]])) + return "resourceManagerTags: string{k:string} expected"; + } return null; }; @@ -7830,6 +8331,13 @@ throw TypeError(".google.cloud.dataproc.v1.ExecutionConfig.authenticationConfig: object expected"); message.authenticationConfig = $root.google.cloud.dataproc.v1.AuthenticationConfig.fromObject(object.authenticationConfig); } + if (object.resourceManagerTags) { + if (typeof object.resourceManagerTags !== "object") + throw TypeError(".google.cloud.dataproc.v1.ExecutionConfig.resourceManagerTags: object expected"); + message.resourceManagerTags = {}; + for (var keys = Object.keys(object.resourceManagerTags), i = 0; i < keys.length; ++i) + message.resourceManagerTags[keys[i]] = String(object.resourceManagerTags[keys[i]]); + } return message; }; @@ -7848,6 +8356,8 @@ var object = {}; if (options.arrays || options.defaults) object.networkTags = []; + if (options.objects || options.defaults) + object.resourceManagerTags = {}; if (options.defaults) { object.serviceAccount = ""; object.kmsKey = ""; @@ -7883,6 +8393,12 @@ object.stagingBucket = message.stagingBucket; if (message.authenticationConfig != null && message.hasOwnProperty("authenticationConfig")) object.authenticationConfig = $root.google.cloud.dataproc.v1.AuthenticationConfig.toObject(message.authenticationConfig, options); + var keys2; + if (message.resourceManagerTags && (keys2 = Object.keys(message.resourceManagerTags)).length) { + object.resourceManagerTags = {}; + for (var j = 0; j < keys2.length; ++j) + object.resourceManagerTags[keys2[j]] = message.resourceManagerTags[keys2[j]]; + } return object; }; @@ -8708,6 +9224,7 @@ * @property {number|Long|null} [shuffleStorageGbSeconds] UsageMetrics shuffleStorageGbSeconds * @property {number|Long|null} [milliAcceleratorSeconds] UsageMetrics milliAcceleratorSeconds * @property {string|null} [acceleratorType] UsageMetrics acceleratorType + * @property {google.protobuf.ITimestamp|null} [updateTime] UsageMetrics updateTime */ /** @@ -8757,6 +9274,14 @@ */ UsageMetrics.prototype.acceleratorType = ""; + /** + * UsageMetrics updateTime. + * @member {google.protobuf.ITimestamp|null|undefined} updateTime + * @memberof google.cloud.dataproc.v1.UsageMetrics + * @instance + */ + UsageMetrics.prototype.updateTime = null; + /** * Creates a new UsageMetrics instance using the specified properties. * @function create @@ -8789,6 +9314,8 @@ writer.uint32(/* id 3, wireType 0 =*/24).int64(message.milliAcceleratorSeconds); if (message.acceleratorType != null && Object.hasOwnProperty.call(message, "acceleratorType")) writer.uint32(/* id 4, wireType 2 =*/34).string(message.acceleratorType); + if (message.updateTime != null && Object.hasOwnProperty.call(message, "updateTime")) + $root.google.protobuf.Timestamp.encode(message.updateTime, writer.uint32(/* id 6, wireType 2 =*/50).fork()).ldelim(); return writer; }; @@ -8841,6 +9368,10 @@ message.acceleratorType = reader.string(); break; } + case 6: { + message.updateTime = $root.google.protobuf.Timestamp.decode(reader, reader.uint32()); + break; + } default: reader.skipType(tag & 7); break; @@ -8888,6 +9419,11 @@ if (message.acceleratorType != null && message.hasOwnProperty("acceleratorType")) if (!$util.isString(message.acceleratorType)) return "acceleratorType: string expected"; + if (message.updateTime != null && message.hasOwnProperty("updateTime")) { + var error = $root.google.protobuf.Timestamp.verify(message.updateTime); + if (error) + return "updateTime." + error; + } return null; }; @@ -8932,6 +9468,11 @@ message.milliAcceleratorSeconds = new $util.LongBits(object.milliAcceleratorSeconds.low >>> 0, object.milliAcceleratorSeconds.high >>> 0).toNumber(); if (object.acceleratorType != null) message.acceleratorType = String(object.acceleratorType); + if (object.updateTime != null) { + if (typeof object.updateTime !== "object") + throw TypeError(".google.cloud.dataproc.v1.UsageMetrics.updateTime: object expected"); + message.updateTime = $root.google.protobuf.Timestamp.fromObject(object.updateTime); + } return message; }; @@ -8965,6 +9506,7 @@ } else object.milliAcceleratorSeconds = options.longs === String ? "0" : 0; object.acceleratorType = ""; + object.updateTime = null; } if (message.milliDcuSeconds != null && message.hasOwnProperty("milliDcuSeconds")) if (typeof message.milliDcuSeconds === "number") @@ -8983,6 +9525,8 @@ object.milliAcceleratorSeconds = options.longs === String ? $util.Long.prototype.toString.call(message.milliAcceleratorSeconds) : options.longs === Number ? new $util.LongBits(message.milliAcceleratorSeconds.low >>> 0, message.milliAcceleratorSeconds.high >>> 0).toNumber() : message.milliAcceleratorSeconds; if (message.acceleratorType != null && message.hasOwnProperty("acceleratorType")) object.acceleratorType = message.acceleratorType; + if (message.updateTime != null && message.hasOwnProperty("updateTime")) + object.updateTime = $root.google.protobuf.Timestamp.toObject(message.updateTime, options); return object; }; @@ -12705,7 +13249,6 @@ * @property {number} HUDI=18 HUDI value * @property {number} ICEBERG=19 ICEBERG value * @property {number} JUPYTER=1 JUPYTER value - * @property {number} JUPYTER_KERNEL_GATEWAY=22 JUPYTER_KERNEL_GATEWAY value * @property {number} PIG=21 PIG value * @property {number} PRESTO=6 PRESTO value * @property {number} TRINO=17 TRINO value @@ -12713,6 +13256,7 @@ * @property {number} SOLR=10 SOLR value * @property {number} ZEPPELIN=4 ZEPPELIN value * @property {number} ZOOKEEPER=8 ZOOKEEPER value + * @property {number} JUPYTER_KERNEL_GATEWAY=22 JUPYTER_KERNEL_GATEWAY value */ v1.Component = (function() { var valuesById = {}, values = Object.create(valuesById); @@ -12727,7 +13271,6 @@ values[valuesById[18] = "HUDI"] = 18; values[valuesById[19] = "ICEBERG"] = 19; values[valuesById[1] = "JUPYTER"] = 1; - values[valuesById[22] = "JUPYTER_KERNEL_GATEWAY"] = 22; values[valuesById[21] = "PIG"] = 21; values[valuesById[6] = "PRESTO"] = 6; values[valuesById[17] = "TRINO"] = 17; @@ -12735,6 +13278,7 @@ values[valuesById[10] = "SOLR"] = 10; values[valuesById[4] = "ZEPPELIN"] = 4; values[valuesById[8] = "ZOOKEEPER"] = 8; + values[valuesById[22] = "JUPYTER_KERNEL_GATEWAY"] = 22; return values; })(); @@ -22426,7 +22970,6 @@ case 18: case 19: case 1: - case 22: case 21: case 6: case 17: @@ -22434,6 +22977,7 @@ case 10: case 4: case 8: + case 22: break; } } @@ -22516,10 +23060,6 @@ case 1: message.optionalComponents[i] = 1; break; - case "JUPYTER_KERNEL_GATEWAY": - case 22: - message.optionalComponents[i] = 22; - break; case "PIG": case 21: message.optionalComponents[i] = 21; @@ -22548,6 +23088,10 @@ case 8: message.optionalComponents[i] = 8; break; + case "JUPYTER_KERNEL_GATEWAY": + case 22: + message.optionalComponents[i] = 22; + break; } } return message; diff --git a/packages/google-cloud-dataproc/protos/protos.json b/packages/google-cloud-dataproc/protos/protos.json index a1e23f9dad7a..5f9b4a396206 100644 --- a/packages/google-cloud-dataproc/protos/protos.json +++ b/packages/google-cloud-dataproc/protos/protos.json @@ -404,7 +404,7 @@ "BatchController": { "options": { "(google.api.default_host)": "dataproc.googleapis.com", - "(google.api.oauth_scopes)": "https://www.googleapis.com/auth/cloud-platform" + "(google.api.oauth_scopes)": "https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/dataproc,https://www.googleapis.com/auth/dataproc.read-only" }, "methods": { "CreateBatch": { @@ -581,7 +581,10 @@ "batches": { "rule": "repeated", "type": "Batch", - "id": 1 + "id": 1, + "options": { + "(google.api.field_behavior)": "OUTPUT_ONLY" + } }, "nextPageToken": { "type": "string", @@ -620,7 +623,8 @@ "pysparkBatch", "sparkBatch", "sparkRBatch", - "sparkSqlBatch" + "sparkSqlBatch", + "pysparkNotebookBatch" ] } }, @@ -674,6 +678,13 @@ "(google.api.field_behavior)": "OPTIONAL" } }, + "pysparkNotebookBatch": { + "type": "PySparkNotebookBatch", + "id": 19, + "options": { + "(google.api.field_behavior)": "OPTIONAL" + } + }, "runtimeInfo": { "type": "RuntimeInfo", "id": 8, @@ -957,6 +968,57 @@ } } }, + "PySparkNotebookBatch": { + "fields": { + "notebookFileUri": { + "type": "string", + "id": 1, + "options": { + "(google.api.field_behavior)": "REQUIRED" + } + }, + "params": { + "keyType": "string", + "type": "string", + "id": 2, + "options": { + "(google.api.field_behavior)": "OPTIONAL" + } + }, + "pythonFileUris": { + "rule": "repeated", + "type": "string", + "id": 3, + "options": { + "(google.api.field_behavior)": "OPTIONAL" + } + }, + "jarFileUris": { + "rule": "repeated", + "type": "string", + "id": 4, + "options": { + "(google.api.field_behavior)": "OPTIONAL" + } + }, + "fileUris": { + "rule": "repeated", + "type": "string", + "id": 5, + "options": { + "(google.api.field_behavior)": "OPTIONAL" + } + }, + "archiveUris": { + "rule": "repeated", + "type": "string", + "id": 6, + "options": { + "(google.api.field_behavior)": "OPTIONAL" + } + } + } + }, "RuntimeConfig": { "fields": { "version": { @@ -1065,7 +1127,8 @@ "type": "string", "id": 7, "options": { - "(google.api.field_behavior)": "OPTIONAL" + "(google.api.field_behavior)": "OPTIONAL", + "(google.api.resource_reference).type": "cloudkms.googleapis.com/CryptoKey" } }, "idleTtl": { @@ -1095,6 +1158,14 @@ "options": { "(google.api.field_behavior)": "OPTIONAL" } + }, + "resourceManagerTags": { + "keyType": "string", + "type": "string", + "id": 12, + "options": { + "(google.api.field_behavior)": "OPTIONAL" + } } } }, @@ -1197,6 +1268,13 @@ "options": { "(google.api.field_behavior)": "OPTIONAL" } + }, + "updateTime": { + "type": "google.protobuf.Timestamp", + "id": 6, + "options": { + "(google.api.field_behavior)": "OPTIONAL" + } } } }, @@ -1424,7 +1502,8 @@ "type": "string", "id": 23, "options": { - "(google.api.field_behavior)": "OPTIONAL" + "(google.api.field_behavior)": "OPTIONAL", + "(google.api.resource_reference).type": "cloudkms.googleapis.com/CryptoKey" } }, "spot": { @@ -1545,14 +1624,14 @@ "HUDI": 18, "ICEBERG": 19, "JUPYTER": 1, - "JUPYTER_KERNEL_GATEWAY": 22, "PIG": 21, "PRESTO": 6, "TRINO": 17, "RANGER": 12, "SOLR": 10, "ZEPPELIN": 4, - "ZOOKEEPER": 8 + "ZOOKEEPER": 8, + "JUPYTER_KERNEL_GATEWAY": 22 } }, "FailureAction": { @@ -5325,7 +5404,7 @@ "SessionTemplateController": { "options": { "(google.api.default_host)": "dataproc.googleapis.com", - "(google.api.oauth_scopes)": "https://www.googleapis.com/auth/cloud-platform" + "(google.api.oauth_scopes)": "https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/dataproc,https://www.googleapis.com/auth/dataproc.read-only" }, "methods": { "CreateSessionTemplate": { @@ -5545,7 +5624,7 @@ "type": "string", "id": 1, "options": { - "(google.api.field_behavior)": "REQUIRED" + "(google.api.field_behavior)": "IDENTIFIER" } }, "description": { @@ -5624,7 +5703,7 @@ "SessionController": { "options": { "(google.api.default_host)": "dataproc.googleapis.com", - "(google.api.oauth_scopes)": "https://www.googleapis.com/auth/cloud-platform" + "(google.api.oauth_scopes)": "https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/dataproc,https://www.googleapis.com/auth/dataproc.read-only" }, "methods": { "CreateSession": { @@ -5897,7 +5976,7 @@ "type": "string", "id": 1, "options": { - "(google.api.field_behavior)": "REQUIRED" + "(google.api.field_behavior)": "IDENTIFIER" } }, "uuid": { diff --git a/packages/google-cloud-dataproc/samples/generated/v1/batch_controller.create_batch.js b/packages/google-cloud-dataproc/samples/generated/v1/batch_controller.create_batch.js index ffc3ad761b69..14b0bb060ba2 100644 --- a/packages/google-cloud-dataproc/samples/generated/v1/batch_controller.create_batch.js +++ b/packages/google-cloud-dataproc/samples/generated/v1/batch_controller.create_batch.js @@ -45,10 +45,9 @@ function main(parent, batch) { /** * Optional. A unique ID used to identify the request. If the service * receives two - * CreateBatchRequest (https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.CreateBatchRequest)s - * with the same request_id, the second request is ignored and the - * Operation that corresponds to the first Batch created and stored - * in the backend is returned. + * `CreateBatchRequests` with the same `request_id`, the second request is + * ignored and the operation that corresponds to the first Batch created and + * stored in the backend is returned. * Recommendation: Set this value to a * UUID (https://en.wikipedia.org/wiki/Universally_unique_identifier). * The value must contain only letters (a-z, A-Z), numbers (0-9), diff --git a/packages/google-cloud-dataproc/samples/generated/v1/batch_controller.list_batches.js b/packages/google-cloud-dataproc/samples/generated/v1/batch_controller.list_batches.js index 082aeb5005df..a2f795f4b2a6 100644 --- a/packages/google-cloud-dataproc/samples/generated/v1/batch_controller.list_batches.js +++ b/packages/google-cloud-dataproc/samples/generated/v1/batch_controller.list_batches.js @@ -48,9 +48,12 @@ function main(parent) { * A filter is a logical expression constraining the values of various fields * in each batch resource. Filters are case sensitive, and may contain * multiple clauses combined with logical operators (AND/OR). - * Supported fields are `batch_id`, `batch_uuid`, `state`, and `create_time`. + * Supported fields are `batch_id`, `batch_uuid`, `state`, `create_time`, and + * `labels`. * e.g. `state = RUNNING and create_time < "2023-01-01T00:00:00Z"` - * filters for batches in state RUNNING that were created before 2023-01-01 + * filters for batches in state RUNNING that were created before 2023-01-01. + * `state = RUNNING and labels.environment=production` filters for batches in + * state in a RUNNING state that have a production environment label. * See https://google.aip.dev/assets/misc/ebnf-filtering.txt for a detailed * description of the filter syntax and a list of supported comparisons. */ diff --git a/packages/google-cloud-dataproc/samples/generated/v1/snippet_metadata_google.cloud.dataproc.v1.json b/packages/google-cloud-dataproc/samples/generated/v1/snippet_metadata_google.cloud.dataproc.v1.json index 97c601873750..29722db4111e 100644 --- a/packages/google-cloud-dataproc/samples/generated/v1/snippet_metadata_google.cloud.dataproc.v1.json +++ b/packages/google-cloud-dataproc/samples/generated/v1/snippet_metadata_google.cloud.dataproc.v1.json @@ -1,7 +1,7 @@ { "clientLibrary": { "name": "nodejs-dataproc", - "version": "6.3.0", + "version": "0.1.0", "language": "TYPESCRIPT", "apis": [ { @@ -234,7 +234,7 @@ "segments": [ { "start": 25, - "end": 78, + "end": 77, "type": "FULL" } ], @@ -326,7 +326,7 @@ "segments": [ { "start": 25, - "end": 85, + "end": 88, "type": "FULL" } ], @@ -375,7 +375,7 @@ "regionTag": "dataproc_v1_generated_BatchController_DeleteBatch_async", "title": "dataproc deleteBatch Sample", "origin": "API_DEFINITION", - "description": " Deletes the batch workload resource. If the batch is not in terminal state, the delete fails and the response returns `FAILED_PRECONDITION`.", + "description": " Deletes the batch workload resource. If the batch is not in a `CANCELLED`, `SUCCEEDED` or `FAILED` [`State`][google.cloud.dataproc.v1.Batch.State], the delete operation fails and the response returns `FAILED_PRECONDITION`.", "canonical": true, "file": "batch_controller.delete_batch.js", "language": "JAVASCRIPT", diff --git a/packages/google-cloud-dataproc/src/index.ts b/packages/google-cloud-dataproc/src/index.ts index 5db1abc6576a..49417fe6517c 100644 --- a/packages/google-cloud-dataproc/src/index.ts +++ b/packages/google-cloud-dataproc/src/index.ts @@ -1,4 +1,4 @@ -// Copyright 2026 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-dataproc/src/v1/batch_controller_client.ts b/packages/google-cloud-dataproc/src/v1/batch_controller_client.ts index 2e9ea824d9d8..129d445657d5 100644 --- a/packages/google-cloud-dataproc/src/v1/batch_controller_client.ts +++ b/packages/google-cloud-dataproc/src/v1/batch_controller_client.ts @@ -378,7 +378,9 @@ export class BatchControllerClient { */ static get scopes() { return [ - 'https://www.googleapis.com/auth/cloud-platform' + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/dataproc', + 'https://www.googleapis.com/auth/dataproc.read-only' ]; } @@ -497,8 +499,10 @@ export class BatchControllerClient { }); } /** - * Deletes the batch workload resource. If the batch is not in terminal state, - * the delete fails and the response returns `FAILED_PRECONDITION`. + * Deletes the batch workload resource. If the batch is not in a + * `CANCELLED`, `SUCCEEDED` or `FAILED` + * {@link protos.google.cloud.dataproc.v1.Batch.State|`State`}, the delete operation fails + * and the response returns `FAILED_PRECONDITION`. * * @param {Object} request * The request object that will be sent. @@ -611,10 +615,9 @@ export class BatchControllerClient { * @param {string} [request.requestId] * Optional. A unique ID used to identify the request. If the service * receives two - * [CreateBatchRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.CreateBatchRequest)s - * with the same request_id, the second request is ignored and the - * Operation that corresponds to the first Batch created and stored - * in the backend is returned. + * `CreateBatchRequests` with the same `request_id`, the second request is + * ignored and the operation that corresponds to the first Batch created and + * stored in the backend is returned. * * Recommendation: Set this value to a * [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). @@ -741,10 +744,13 @@ export class BatchControllerClient { * A filter is a logical expression constraining the values of various fields * in each batch resource. Filters are case sensitive, and may contain * multiple clauses combined with logical operators (AND/OR). - * Supported fields are `batch_id`, `batch_uuid`, `state`, and `create_time`. + * Supported fields are `batch_id`, `batch_uuid`, `state`, `create_time`, and + * `labels`. * * e.g. `state = RUNNING and create_time < "2023-01-01T00:00:00Z"` - * filters for batches in state RUNNING that were created before 2023-01-01 + * filters for batches in state RUNNING that were created before 2023-01-01. + * `state = RUNNING and labels.environment=production` filters for batches in + * state in a RUNNING state that have a production environment label. * * See https://google.aip.dev/assets/misc/ebnf-filtering.txt for a detailed * description of the filter syntax and a list of supported comparisons. @@ -862,10 +868,13 @@ export class BatchControllerClient { * A filter is a logical expression constraining the values of various fields * in each batch resource. Filters are case sensitive, and may contain * multiple clauses combined with logical operators (AND/OR). - * Supported fields are `batch_id`, `batch_uuid`, `state`, and `create_time`. + * Supported fields are `batch_id`, `batch_uuid`, `state`, `create_time`, and + * `labels`. * * e.g. `state = RUNNING and create_time < "2023-01-01T00:00:00Z"` - * filters for batches in state RUNNING that were created before 2023-01-01 + * filters for batches in state RUNNING that were created before 2023-01-01. + * `state = RUNNING and labels.environment=production` filters for batches in + * state in a RUNNING state that have a production environment label. * * See https://google.aip.dev/assets/misc/ebnf-filtering.txt for a detailed * description of the filter syntax and a list of supported comparisons. @@ -932,10 +941,13 @@ export class BatchControllerClient { * A filter is a logical expression constraining the values of various fields * in each batch resource. Filters are case sensitive, and may contain * multiple clauses combined with logical operators (AND/OR). - * Supported fields are `batch_id`, `batch_uuid`, `state`, and `create_time`. + * Supported fields are `batch_id`, `batch_uuid`, `state`, `create_time`, and + * `labels`. * * e.g. `state = RUNNING and create_time < "2023-01-01T00:00:00Z"` - * filters for batches in state RUNNING that were created before 2023-01-01 + * filters for batches in state RUNNING that were created before 2023-01-01. + * `state = RUNNING and labels.environment=production` filters for batches in + * state in a RUNNING state that have a production environment label. * * See https://google.aip.dev/assets/misc/ebnf-filtering.txt for a detailed * description of the filter syntax and a list of supported comparisons. diff --git a/packages/google-cloud-dataproc/src/v1/session_controller_client.ts b/packages/google-cloud-dataproc/src/v1/session_controller_client.ts index 4bd0dc14e797..892bd1fab160 100644 --- a/packages/google-cloud-dataproc/src/v1/session_controller_client.ts +++ b/packages/google-cloud-dataproc/src/v1/session_controller_client.ts @@ -394,7 +394,9 @@ export class SessionControllerClient { */ static get scopes() { return [ - 'https://www.googleapis.com/auth/cloud-platform' + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/dataproc', + 'https://www.googleapis.com/auth/dataproc.read-only' ]; } diff --git a/packages/google-cloud-dataproc/src/v1/session_template_controller_client.ts b/packages/google-cloud-dataproc/src/v1/session_template_controller_client.ts index 3bc0a772c4a1..ff1e70778c48 100644 --- a/packages/google-cloud-dataproc/src/v1/session_template_controller_client.ts +++ b/packages/google-cloud-dataproc/src/v1/session_template_controller_client.ts @@ -344,7 +344,9 @@ export class SessionTemplateControllerClient { */ static get scopes() { return [ - 'https://www.googleapis.com/auth/cloud-platform' + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/dataproc', + 'https://www.googleapis.com/auth/dataproc.read-only' ]; } diff --git a/packages/google-cloud-dataproc/webpack.config.js b/packages/google-cloud-dataproc/webpack.config.js index e6ae2897925d..050b272c0e03 100644 --- a/packages/google-cloud-dataproc/webpack.config.js +++ b/packages/google-cloud-dataproc/webpack.config.js @@ -1,4 +1,4 @@ -// Copyright 2026 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License.