diff --git a/behave_framework/src/minifi_behave/steps/checking_steps.py b/behave_framework/src/minifi_behave/steps/checking_steps.py index 5f52ee7408..418e875c93 100644 --- a/behave_framework/src/minifi_behave/steps/checking_steps.py +++ b/behave_framework/src/minifi_behave/steps/checking_steps.py @@ -119,7 +119,7 @@ def verify_minifi_logs_match_regex(context, regex, duration): @step('no errors were generated on the http-proxy regarding "{url}"') def verify_no_errors_on_http_proxy(context: MinifiTestContext, url: str): http_proxy_container = next(container for container in context.containers.values() if isinstance(container, HttpProxy)) - assert http_proxy_container.check_http_proxy_access(url) or http_proxy_container.log_app_output() + assert http_proxy_container.check_http_proxy_access(url) or log_due_to_failure(context) @then('in the "{container}" container no files are placed in the "{directory}" directory in {duration} of running time') diff --git a/core-framework/common/include/core/PropertyDefinitionBuilder.h b/core-framework/common/include/core/PropertyDefinitionBuilder.h index a9640837ee..d206720e84 100644 --- a/core-framework/common/include/core/PropertyDefinitionBuilder.h +++ b/core-framework/common/include/core/PropertyDefinitionBuilder.h @@ -27,7 +27,21 @@ namespace org::apache::nifi::minifi::core { namespace detail { template inline constexpr auto TypeNames = std::array{core::className()...}; -} + +template +struct StringLiteral { + char value[N]; + constexpr StringLiteral(const char (&str)[N]) { // NOLINT(runtime/explicit) + for (size_t i = 0; i < N; ++i) { + value[i] = str[i]; + } + } +}; + +// A variable template that creates permanent static memory for the span to point to +template +inline constexpr auto StaticAllowedType = std::array{std::string_view{str.value, sizeof(str.value) - 1}}; +} // namespace detail template struct PropertyDefinitionBuilder { @@ -81,6 +95,12 @@ struct PropertyDefinitionBuilder { return *this; } + template + constexpr PropertyDefinitionBuilder withAllowedType() { + property.allowed_types = detail::StaticAllowedType; + return *this; + } + constexpr PropertyDefinitionBuilder withValidator(const PropertyValidator& property_validator) { #if defined(__clang__) && (__clang_major__ <= 18) property.validator = &property_validator; diff --git a/core-framework/include/utils/AttributeErrors.h b/core-framework/common/include/utils/AttributeErrors.h similarity index 100% rename from core-framework/include/utils/AttributeErrors.h rename to core-framework/common/include/utils/AttributeErrors.h diff --git a/core-framework/include/utils/RegexUtils.h b/core-framework/common/include/utils/RegexUtils.h similarity index 100% rename from core-framework/include/utils/RegexUtils.h rename to core-framework/common/include/utils/RegexUtils.h diff --git a/libminifi/src/utils/AttributeErrors.cpp b/core-framework/common/src/utils/AttributeErrors.cpp similarity index 100% rename from libminifi/src/utils/AttributeErrors.cpp rename to core-framework/common/src/utils/AttributeErrors.cpp diff --git a/core-framework/src/utils/RegexUtils.cpp b/core-framework/common/src/utils/RegexUtils.cpp similarity index 100% rename from core-framework/src/utils/RegexUtils.cpp rename to core-framework/common/src/utils/RegexUtils.cpp diff --git a/extension-framework/cpp-extension-lib/CMakeLists.txt b/extension-framework/cpp-extension-lib/CMakeLists.txt index 6b22dd980a..f1161279bf 100644 --- a/extension-framework/cpp-extension-lib/CMakeLists.txt +++ b/extension-framework/cpp-extension-lib/CMakeLists.txt @@ -24,4 +24,4 @@ target_include_directories(minifi-cpp-extension-lib PUBLIC include) target_link_libraries(minifi-cpp-extension-lib PUBLIC minifi-core-framework-common minifi-c-api) add_subdirectory(libtest) - +add_subdirectory(mocklib) diff --git a/extension-framework/cpp-extension-lib/include/api/core/FlowFile.h b/extension-framework/cpp-extension-lib/include/api/core/FlowFile.h index 4a288f36d7..da9e6e8e13 100644 --- a/extension-framework/cpp-extension-lib/include/api/core/FlowFile.h +++ b/extension-framework/cpp-extension-lib/include/api/core/FlowFile.h @@ -17,8 +17,10 @@ */ #pragma once +#include #include #include + #include "minifi-c.h" namespace org::apache::nifi::minifi::api::core { diff --git a/extension-framework/cpp-extension-lib/include/api/core/ProcessContext.h b/extension-framework/cpp-extension-lib/include/api/core/ProcessContext.h index 6f012d719a..26a55a44a6 100644 --- a/extension-framework/cpp-extension-lib/include/api/core/ProcessContext.h +++ b/extension-framework/cpp-extension-lib/include/api/core/ProcessContext.h @@ -1,5 +1,5 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one or more + * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 @@ -17,26 +17,52 @@ #pragma once -#include #include +#include +#include "api/core/FlowFile.h" +#include "api/utils/Proxy.h" +#include "api/utils/Ssl.h" #include "minifi-c.h" #include "minifi-cpp/core/PropertyDefinition.h" -#include "api/core/FlowFile.h" namespace org::apache::nifi::minifi::api::core { class ProcessContext { public: - explicit ProcessContext(MinifiProcessContext* impl): impl_(impl) {} + virtual ~ProcessContext() noexcept = default; + + ProcessContext() = default; + ProcessContext(const ProcessContext&) = delete; + ProcessContext(ProcessContext&&) = delete; + ProcessContext& operator=(const ProcessContext&) = delete; + ProcessContext& operator=(ProcessContext&&) = delete; + + [[nodiscard]] virtual std::expected getProperty(const minifi::core::PropertyReference& prop, + const FlowFile* ff) const = 0; + [[nodiscard]] virtual std::expected getControllerService(const minifi::core::PropertyReference& prop) const = 0; + [[nodiscard]] virtual bool hasNonEmptyProperty(std::string_view name) const = 0; + [[nodiscard]] virtual std::map getDynamicProperties(const FlowFile* flow_file) const = 0; + + [[nodiscard]] virtual std::expected, std::error_code> getSslData(const minifi::core::PropertyReference& prop) const = 0; + [[nodiscard]] virtual std::expected, std::error_code> getProxyData(const minifi::core::PropertyReference& prop) const = 0; +}; + +class CffiProcessContext : public ProcessContext { + public: + explicit CffiProcessContext(MinifiProcessContext* impl) : impl_(impl) {} + + [[nodiscard]] std::expected getProperty(const minifi::core::PropertyReference& property_reference, + const FlowFile* flow_file) const override; + [[nodiscard]] std::expected getControllerService(const minifi::core::PropertyReference& prop) const override; + [[nodiscard]] std::map getDynamicProperties(const FlowFile* flow_file) const override; + [[nodiscard]] bool hasNonEmptyProperty(std::string_view name) const override; - std::expected getProperty(std::string_view name, const FlowFile* flow_file = nullptr) const; - std::expected getProperty(const minifi::core::PropertyReference& property_reference, const FlowFile* flow_file = nullptr) const { - return getProperty(property_reference.name, flow_file); - } - [[nodiscard]] std::expected getControllerService(std::string_view controller_service_name, std::string_view controller_service_class) const; + [[nodiscard]] std::expected, std::error_code> getSslData(const minifi::core::PropertyReference& prop) const override; + [[nodiscard]] std::expected, std::error_code> getProxyData(const minifi::core::PropertyReference& prop) const override; - [[nodiscard]] bool hasNonEmptyProperty(std::string_view name) const; + private: + [[nodiscard]] std::expected getProperty(std::string_view name, const FlowFile* flow_file) const; private: MinifiProcessContext* impl_; diff --git a/extension-framework/cpp-extension-lib/include/api/core/ProcessSession.h b/extension-framework/cpp-extension-lib/include/api/core/ProcessSession.h index e3e8122497..1ef50770d9 100644 --- a/extension-framework/cpp-extension-lib/include/api/core/ProcessSession.h +++ b/extension-framework/cpp-extension-lib/include/api/core/ProcessSession.h @@ -29,23 +29,54 @@ namespace org::apache::nifi::minifi::api::core { class ProcessSession { public: - explicit ProcessSession(MinifiProcessSession* impl): impl_(impl) {} + virtual ~ProcessSession() = default; - FlowFile create(const FlowFile* parent = nullptr); - FlowFile get(); - void transfer(FlowFile ff, const minifi::core::Relationship& relationship); - void remove(FlowFile ff); - void write(FlowFile& flow, const io::OutputStreamCallback& callback); - void read(FlowFile& flow, const io::InputStreamCallback& callback); + ProcessSession() = default; - void setAttribute(FlowFile& ff, std::string_view key, std::string value); - void removeAttribute(FlowFile& ff, std::string_view key); - std::optional getAttribute(FlowFile& ff, std::string_view key); - std::map getAttributes(FlowFile& ff); + ProcessSession(const ProcessSession&) = delete; + ProcessSession(ProcessSession&&) = delete; + ProcessSession& operator=(const ProcessSession&) = delete; + ProcessSession& operator=(ProcessSession&&) = delete; + + virtual FlowFile create(const FlowFile* parent = nullptr) = 0; + virtual FlowFile get() = 0; + + virtual void penalize(FlowFile& ff) = 0; + virtual void transfer(FlowFile ff, const minifi::core::Relationship& relationship) = 0; + virtual void remove(FlowFile ff) = 0; + virtual void write(FlowFile& flow, const io::OutputStreamCallback& callback) = 0; + virtual void read(FlowFile& flow, const io::InputStreamCallback& callback) = 0; + + virtual void setAttribute(FlowFile& ff, std::string_view key, std::string value) = 0; + virtual void removeAttribute(FlowFile& ff, std::string_view key) = 0; + [[nodiscard]] virtual std::optional getAttribute(FlowFile& ff, std::string_view key) = 0; + [[nodiscard]] virtual std::map getAttributes(const FlowFile& ff) const = 0; + [[nodiscard]] virtual std::string getFlowFileId(const FlowFile& ff) const = 0; + [[nodiscard]] virtual uint64_t getFlowFileSize(const FlowFile& ff) const = 0; void writeBuffer(FlowFile& flow_file, std::span buffer); void writeBuffer(FlowFile& flow_file, std::span buffer); - std::vector readBuffer(FlowFile& flow_file); + [[nodiscard]] std::vector readBuffer(FlowFile& flow_file); +}; + +class CffiProcessSession : public ProcessSession { + public: + explicit CffiProcessSession(MinifiProcessSession* impl): impl_(impl) {} + + FlowFile create(const FlowFile* parent = nullptr) override; + FlowFile get() override; + void penalize(FlowFile& ff) override; + void transfer(FlowFile ff, const minifi::core::Relationship& relationship) override; + void remove(FlowFile ff) override; + void write(FlowFile& flow, const io::OutputStreamCallback& callback) override; + void read(FlowFile& flow, const io::InputStreamCallback& callback) override; + + void setAttribute(FlowFile& ff, std::string_view key, std::string value) override; + void removeAttribute(FlowFile& ff, std::string_view key) override; + [[nodiscard]] std::optional getAttribute(FlowFile& ff, std::string_view key) override; + [[nodiscard]] std::map getAttributes(const FlowFile& ff) const override; + [[nodiscard]] std::string getFlowFileId(const FlowFile& ff) const override; + [[nodiscard]] uint64_t getFlowFileSize(const FlowFile& ff) const override; private: MinifiProcessSession* impl_; diff --git a/extension-framework/cpp-extension-lib/include/api/core/Resource.h b/extension-framework/cpp-extension-lib/include/api/core/Resource.h index 8187605e51..60766741cb 100644 --- a/extension-framework/cpp-extension-lib/include/api/core/Resource.h +++ b/extension-framework/cpp-extension-lib/include/api/core/Resource.h @@ -49,17 +49,17 @@ void useProcessorClassDefinition(Fn&& fn) { std::vector dynamic_properties; for (auto& prop : Class::DynamicProperties) { dynamic_properties.push_back(MinifiDynamicPropertyDefinition { - .name = utils::toStringView(prop.name), - .value = utils::toStringView(prop.value), - .description = utils::toStringView(prop.description), + .name = utils::minifiStringView(prop.name), + .value = utils::minifiStringView(prop.value), + .description = utils::minifiStringView(prop.description), .supports_expression_language = prop.supports_expression_language }); } std::vector relationships; for (auto& rel : Class::Relationships) { relationships.push_back(MinifiRelationshipDefinition{ - .name = utils::toStringView(rel.name), - .description = utils::toStringView(rel.description) + .name = utils::minifiStringView(rel.name), + .description = utils::minifiStringView(rel.description) }); } std::vector> attribute_relationships_cache; @@ -67,20 +67,20 @@ void useProcessorClassDefinition(Fn&& fn) { for (auto& attr : Class::OutputAttributes) { std::vector rel_cache; for (auto& rel : attr.relationships) { - rel_cache.push_back(utils::toStringView(rel.name)); + rel_cache.push_back(utils::minifiStringView(rel.name)); } output_attributes.push_back(MinifiOutputAttributeDefinition { - .name = utils::toStringView(attr.name), + .name = utils::minifiStringView(attr.name), .relationships_count = gsl::narrow(attr.relationships.size()), .relationships_ptr = rel_cache.data(), - .description = utils::toStringView(attr.description) + .description = utils::minifiStringView(attr.description) }); attribute_relationships_cache.push_back(std::move(rel_cache)); } MinifiProcessorClassDefinition definition{ - .full_name = utils::toStringView(full_name), - .description = utils::toStringView(Class::Description), + .full_name = utils::minifiStringView(full_name), + .description = utils::minifiStringView(Class::Description), .class_properties_count = gsl::narrow(class_properties.size()), .class_properties_ptr = class_properties.data(), .dynamic_properties_count = gsl::narrow(dynamic_properties.size()), @@ -100,7 +100,7 @@ void useProcessorClassDefinition(Fn&& fn) { return new Class{minifi::core::ProcessorMetadata{ .uuid = minifi::utils::Identifier::parse(std::string{metadata.uuid.data, metadata.uuid.length}).value(), .name = std::string{metadata.name.data, metadata.name.length}, - .logger = std::make_shared(metadata.logger)}}; + .logger = std::make_shared(metadata.logger)}}; } catch (...) { return nullptr; } }, .destroy = [] (MINIFI_OWNED void* self) -> void { @@ -110,8 +110,8 @@ void useProcessorClassDefinition(Fn&& fn) { return static_cast(self)->getTriggerWhenEmpty(); }, .onTrigger = [] (void* self, MinifiProcessContext* context, MinifiProcessSession* session) -> MinifiStatus { - ProcessContext context_wrapper(context); - ProcessSession session_wrapper(session); + CffiProcessContext context_wrapper(context); + CffiProcessSession session_wrapper(session); try { return static_cast(self)->onTrigger(context_wrapper, session_wrapper); } catch (...) { @@ -119,7 +119,7 @@ void useProcessorClassDefinition(Fn&& fn) { } }, .onSchedule = [] (void* self, MinifiProcessContext* context) -> MinifiStatus { - ProcessContext context_wrapper(context); + CffiProcessContext context_wrapper(context); try { return static_cast(self)->onSchedule(context_wrapper); } catch (...) { @@ -136,7 +136,7 @@ void useProcessorClassDefinition(Fn&& fn) { std::vector names; std::vector values; for (auto& [name, val] : metrics) { - names.push_back(utils::toStringView(name)); + names.push_back(utils::minifiStringView(name)); values.push_back(val); } return MinifiPublishedMetricsCreate(gsl::narrow(metrics.size()), names.data(), values.data()); @@ -155,8 +155,8 @@ void useControllerServiceClassDefinition(Fn&& fn) { std::vector class_properties = utils::toProperties(Class::Properties, string_vector_cache); - MinifiControllerServiceClassDefinition definition{.full_name = utils::toStringView(full_name), - .description = utils::toStringView(Class::Description), + MinifiControllerServiceClassDefinition definition{.full_name = utils::minifiStringView(full_name), + .description = utils::minifiStringView(Class::Description), .class_properties_count = gsl::narrow(class_properties.size()), .class_properties_ptr = class_properties.data(), @@ -166,7 +166,7 @@ void useControllerServiceClassDefinition(Fn&& fn) { return new Class{minifi::core::ControllerServiceMetadata{ .uuid = minifi::utils::Identifier::parse(std::string{metadata.uuid.data, metadata.uuid.length}).value(), .name = std::string{metadata.name.data, metadata.name.length}, - .logger = std::make_shared(metadata.logger)}}; + .logger = std::make_shared(metadata.logger)}}; } catch (...) { return nullptr; } }, .destroy = [](MINIFI_OWNED void* self) -> void { delete static_cast(self); }, @@ -186,4 +186,18 @@ void useControllerServiceClassDefinition(Fn&& fn) { fn(definition); } +template +void registerProcessors(MinifiExtension* extension) { + (core::useProcessorClassDefinition([&](const MinifiProcessorClassDefinition& definition) { + MinifiRegisterProcessor(extension, &definition); + }), ...); +} + +template +void registerControllerServices(MinifiExtension* extension) { + (core::useControllerServiceClassDefinition([&](const MinifiControllerServiceClassDefinition& definition) { + MinifiRegisterControllerService(extension, &definition); + }), ...); +} + } // namespace org::apache::nifi::minifi::api::core diff --git a/extension-framework/cpp-extension-lib/include/api/core/logging/Logger.h b/extension-framework/cpp-extension-lib/include/api/core/logging/Logger.h index ada9e51785..5a44013445 100644 --- a/extension-framework/cpp-extension-lib/include/api/core/logging/Logger.h +++ b/extension-framework/cpp-extension-lib/include/api/core/logging/Logger.h @@ -20,15 +20,14 @@ #include #include -#include "fmt/chrono.h" #include "minifi-c.h" #include "minifi-cpp/core/logging/Logger.h" namespace org::apache::nifi::minifi::api::core::logging { -class Logger : public minifi::core::logging::Logger { +class CffiLogger : public minifi::core::logging::Logger { public: - explicit Logger(MinifiLogger* impl): impl_(impl) {} + explicit CffiLogger(MinifiLogger* impl): impl_(impl) {} void set_max_log_size(int size) override; void log_string(minifi::core::logging::LOG_LEVEL level, std::string str) override; diff --git a/extension-framework/cpp-extension-lib/include/api/utils/ProcessorConfigUtils.h b/extension-framework/cpp-extension-lib/include/api/utils/ProcessorConfigUtils.h index a73dd75498..2ec9571edd 100644 --- a/extension-framework/cpp-extension-lib/include/api/utils/ProcessorConfigUtils.h +++ b/extension-framework/cpp-extension-lib/include/api/utils/ProcessorConfigUtils.h @@ -32,47 +32,47 @@ namespace org::apache::nifi::minifi::api::utils { inline std::string parseProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - return ctx.getProperty(property.name, flow_file) + return ctx.getProperty(property, flow_file) | minifi::utils::orThrow(fmt::format("Expected valid value from \"{}\"", property.name)); } inline bool parseBoolProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - return ctx.getProperty(property.name, flow_file) + return ctx.getProperty(property, flow_file) | minifi::utils::andThen(parsing::parseBool) | minifi::utils::orThrow(fmt::format("Expected parsable bool from \"{}\"", property.name)); } inline uint64_t parseU64Property(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - return ctx.getProperty(property.name, flow_file) + return ctx.getProperty(property, flow_file) | minifi::utils::andThen(parsing::parseIntegral) | minifi::utils::orThrow(fmt::format("Expected parsable uint64_t from \"{}\"", property.name)); } inline int64_t parseI64Property(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - return ctx.getProperty(property.name, flow_file) + return ctx.getProperty(property, flow_file) | minifi::utils::andThen(parsing::parseIntegral) | minifi::utils::orThrow(fmt::format("Expected parsable int64_t from \"{}\"", property.name)); } inline std::chrono::milliseconds parseDurationProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - return ctx.getProperty(property.name, flow_file) + return ctx.getProperty(property, flow_file) | minifi::utils::andThen(parsing::parseDuration) | minifi::utils::orThrow(fmt::format("Expected parsable duration from \"{}\"", property.name)); } inline uint64_t parseDataSizeProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - return ctx.getProperty(property.name, flow_file) + return ctx.getProperty(property, flow_file) | minifi::utils::andThen(parsing::parseDataSize) | minifi::utils::orThrow(fmt::format("Expected parsable data size from \"{}\"", property.name)); } inline std::optional parseOptionalProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - return ctx.getProperty(property.name, flow_file) + return ctx.getProperty(property, flow_file) | minifi::utils::toOptional(); } inline std::optional parseOptionalBoolProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - if (const auto property_str = ctx.getProperty(property.name, flow_file)) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { return parsing::parseBool(*property_str) | minifi::utils::orThrow(fmt::format("Expected parsable bool from \"{}\"", property.name)); } @@ -80,7 +80,7 @@ inline std::optional parseOptionalBoolProperty(const core::ProcessContext& } inline std::optional parseOptionalU64Property(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - if (const auto property_str = ctx.getProperty(property.name, flow_file)) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { if (property_str->empty()) { return std::nullopt; } @@ -92,7 +92,7 @@ inline std::optional parseOptionalU64Property(const core::ProcessConte } inline std::optional parseOptionalI64Property(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - if (const auto property_str = ctx.getProperty(property.name, flow_file)) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { if (property_str->empty()) { return std::nullopt; } @@ -106,7 +106,7 @@ inline std::optional parseOptionalI64Property(const core::ProcessContex inline std::optional parseOptionalDurationProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - if (const auto property_str = ctx.getProperty(property.name, flow_file)) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { if (property_str->empty()) { return std::nullopt; } @@ -118,7 +118,7 @@ inline std::optional parseOptionalDurationProperty(co } inline std::optional parseOptionalDataSizeProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - if (const auto property_str = ctx.getProperty(property.name, flow_file)) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { if (property_str->empty()) { return std::nullopt; } @@ -130,7 +130,7 @@ inline std::optional parseOptionalDataSizeProperty(const core::Process } inline std::optional parseOptionalFloatProperty(const core::ProcessContext& ctx, const minifi::core::PropertyReference& property, const core::FlowFile* flow_file = nullptr) { - if (const auto property_str = ctx.getProperty(property.name, flow_file)) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { if (property_str->empty()) { return std::nullopt; } @@ -142,7 +142,7 @@ inline std::optional parseOptionalFloatProperty(const core::ProcessContex template T parseEnumProperty(const core::ProcessContext& context, const minifi::core::PropertyReference& prop, const core::FlowFile* flow_file = nullptr) { - const auto enum_str = context.getProperty(prop.name, flow_file); + const auto enum_str = context.getProperty(prop, flow_file); if (!enum_str) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' is missing"); } @@ -155,7 +155,7 @@ T parseEnumProperty(const core::ProcessContext& context, const minifi::core::Pro template std::optional parseOptionalEnumProperty(const core::ProcessContext& context, const minifi::core::PropertyReference& prop) { - const auto enum_str = context.getProperty(prop.name); + const auto enum_str = context.getProperty(prop, nullptr); if (!enum_str) { return std::nullopt; @@ -169,12 +169,8 @@ std::optional parseOptionalEnumProperty(const core::ProcessContext& context, template ControllerServiceType* parseOptionalControllerService(const core::ProcessContext& context, const minifi::core::PropertyReference& prop) { - const auto controller_service_name = context.getProperty(prop.name); - if (!controller_service_name || controller_service_name->empty()) { - return nullptr; - } + auto service = context.getControllerService(prop); - auto service = context.getControllerService(*controller_service_name, minifi::core::className()); if (!service) { return nullptr; } diff --git a/extension-framework/cpp-extension-lib/include/api/utils/Proxy.h b/extension-framework/cpp-extension-lib/include/api/utils/Proxy.h new file mode 100644 index 0000000000..53ba58725a --- /dev/null +++ b/extension-framework/cpp-extension-lib/include/api/utils/Proxy.h @@ -0,0 +1,42 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +namespace org::apache::nifi::minifi::api::utils { + +enum class ProxyType { + DIRECT, + HTTP +}; + +struct BasicAuthCredentials { + std::string username; + std::string password; +}; + +struct ProxyData { + std::string host; + uint16_t port; + std::optional proxy_credentials; + ProxyType proxy_type; +}; + +} // namespace org::apache::nifi::minifi::api::utils diff --git a/extension-framework/cpp-extension-lib/include/api/utils/Ssl.h b/extension-framework/cpp-extension-lib/include/api/utils/Ssl.h new file mode 100644 index 0000000000..7ffdcb4a9f --- /dev/null +++ b/extension-framework/cpp-extension-lib/include/api/utils/Ssl.h @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +namespace org::apache::nifi::minifi::api::utils::net { + +enum class ClientAuthOption { + NONE, + WANT, + REQUIRED +}; + +struct SslData { + std::filesystem::path ca_loc; + std::filesystem::path cert_loc; + std::filesystem::path key_loc; + std::string key_pw; + + bool isValid() const { + return !cert_loc.empty() && !key_loc.empty(); + } +}; + +struct SslServerOptions { + SslData cert_data; + ClientAuthOption client_auth_option; + + SslServerOptions(SslData cert_data, ClientAuthOption client_auth_option) + : cert_data(cert_data), + client_auth_option(client_auth_option) {} +}; + + +} // namespace org::apache::nifi::minifi::api::utils::net diff --git a/extension-framework/cpp-extension-lib/include/api/utils/minifi-c-utils.h b/extension-framework/cpp-extension-lib/include/api/utils/minifi-c-utils.h index 43fb283067..227ad4c808 100644 --- a/extension-framework/cpp-extension-lib/include/api/utils/minifi-c-utils.h +++ b/extension-framework/cpp-extension-lib/include/api/utils/minifi-c-utils.h @@ -32,10 +32,18 @@ namespace org::apache::nifi::minifi::api::utils { -inline MinifiStringView toStringView(std::string_view str) { +inline MinifiStringView minifiStringView(const std::string_view str) { return MinifiStringView{.data = str.data(), .length = str.length()}; } +inline std::string toString(const MinifiStringView sv) { + return {sv.data, sv.length}; +} + +inline std::string_view toStringView(const MinifiStringView sv) { + return {sv.data, sv.length}; +} + template std::string classNameWithDots() { std::string class_name{minifi::core::className()}; @@ -59,27 +67,27 @@ inline std::vector toProperties(std::span sv_cache; const size_t allowed_values_begin = sv_cache.size(); for (auto& allowed_value : prop.allowed_values) { - sv_cache.emplace_back(toStringView(allowed_value)); + sv_cache.emplace_back(minifiStringView(allowed_value)); } const std::optional allowed_types_begin = [&] () -> std::optional { if (prop.allowed_types.empty()) { return std::nullopt; } gsl_Expects(prop.allowed_types.size() == 1); - sv_cache.emplace_back(toStringView(prop.allowed_types[0])); + sv_cache.emplace_back(minifiStringView(prop.allowed_types[0])); return sv_cache.size() - 1; }(); const std::optional default_value_begin = [&] () -> std::optional { if (!prop.default_value) { return std::nullopt; } - sv_cache.emplace_back(toStringView(*prop.default_value)); + sv_cache.emplace_back(minifiStringView(*prop.default_value)); return sv_cache.size() - 1; }(); properties.push_back(MinifiPropertyDefinition{ - .name = toStringView(prop.name), - .display_name = toStringView(prop.display_name), - .description = toStringView(prop.description), + .name = minifiStringView(prop.name), + .display_name = minifiStringView(prop.display_name), + .description = minifiStringView(prop.description), .is_required = prop.is_required, .is_sensitive = prop.is_sensitive, diff --git a/extension-framework/cpp-extension-lib/mocklib/CMakeLists.txt b/extension-framework/cpp-extension-lib/mocklib/CMakeLists.txt new file mode 100644 index 0000000000..f93870f375 --- /dev/null +++ b/extension-framework/cpp-extension-lib/mocklib/CMakeLists.txt @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +file(GLOB SRC_FILES CONFIGURE_DEPENDS "src/*.cpp") + +add_library(libmock-minifi STATIC ${SRC_FILES}) +target_link_libraries(libmock-minifi PUBLIC minifi-cpp-extension-lib minifi-core-framework-common minifi-c-api) +target_include_directories(libmock-minifi PUBLIC include) diff --git a/extension-framework/cpp-extension-lib/mocklib/include/MockLogger.h b/extension-framework/cpp-extension-lib/mocklib/include/MockLogger.h new file mode 100644 index 0000000000..2c54cc6635 --- /dev/null +++ b/extension-framework/cpp-extension-lib/mocklib/include/MockLogger.h @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "minifi-cpp/core/logging/Logger.h" + +namespace org::apache::nifi::minifi::mock { +class MockLogger : public core::logging::Logger { + public: + void set_max_log_size(int) override {} + void log_string(const core::logging::LOG_LEVEL level, std::string s) override { logs_[level].emplace_back(std::move(s)); } + [[nodiscard]] bool should_log(const core::logging::LOG_LEVEL level) override { return level >= log_level_; } + [[nodiscard]] core::logging::LOG_LEVEL level() const override { return log_level_; } + + core::logging::LOG_LEVEL log_level_ = core::logging::LOG_LEVEL::trace; + std::map> logs_; +}; +} // namespace org::apache::nifi::minifi::mock diff --git a/extension-framework/cpp-extension-lib/mocklib/include/MockProcessContext.h b/extension-framework/cpp-extension-lib/mocklib/include/MockProcessContext.h new file mode 100644 index 0000000000..cddcc97610 --- /dev/null +++ b/extension-framework/cpp-extension-lib/mocklib/include/MockProcessContext.h @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#include "MockUtils.h" +#include "api/core/ProcessContext.h" + +namespace org::apache::nifi::minifi::mock { +class MockProcessContext : public api::core::ProcessContext { + public: + using ProcessContext::ProcessContext; + + [[nodiscard]] std::expected getProperty(const core::PropertyReference& property_reference, + const api::core::FlowFile* flow_file) const override; + [[nodiscard]] std::expected getControllerService(const minifi::core::PropertyReference& prop) const override; + [[nodiscard]] std::map getDynamicProperties(const api::core::FlowFile* flow_file) const override; + [[nodiscard]] bool hasNonEmptyProperty(std::string_view name) const override; + + [[nodiscard]] std::expected, std::error_code> getSslData(const minifi::core::PropertyReference& prop) const override; + [[nodiscard]] std::expected, std::error_code> getProxyData(const minifi::core::PropertyReference& prop) const override; + + std::map> properties_; + + private: + [[nodiscard]] std::expected getProperty(std::string_view name, + const api::core::FlowFile* flow_file) const; +}; + +} // namespace org::apache::nifi::minifi::mock diff --git a/extension-framework/cpp-extension-lib/mocklib/include/MockProcessSession.h b/extension-framework/cpp-extension-lib/mocklib/include/MockProcessSession.h new file mode 100644 index 0000000000..f735c8f149 --- /dev/null +++ b/extension-framework/cpp-extension-lib/mocklib/include/MockProcessSession.h @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#include "MockUtils.h" +#include "api/core/ProcessSession.h" +#include "range/v3/range/conversion.hpp" +#include "range/v3/view/transform.hpp" + +struct MinifiFlowFile { + explicit MinifiFlowFile(std::string content_str) { + this->content = content_str + | ranges::views::transform([](char c) { return static_cast(c); }) + | ranges::to>(); + } + + ~MinifiFlowFile() = default; + + MinifiFlowFile() = default; + MinifiFlowFile(const MinifiFlowFile&) = default; + MinifiFlowFile(MinifiFlowFile&&) = default; + MinifiFlowFile& operator=(const MinifiFlowFile&) = default; + MinifiFlowFile& operator=(MinifiFlowFile&&) = default; + + std::map attributes; + std::vector content; + bool is_penalized = false; + std::string id; +}; + +namespace org::apache::nifi::minifi::mock { + +class MockProcessSession : public api::core::ProcessSession { + public: + MockProcessSession() = default; + MockProcessSession(const MockProcessSession&) = delete; + MockProcessSession& operator=(const MockProcessSession&) = delete; + MockProcessSession(MockProcessSession&&) = delete; + MockProcessSession& operator=(MockProcessSession&&) = delete; + + api::core::FlowFile create(const api::core::FlowFile* parent) override; + api::core::FlowFile get() override; + void penalize(api::core::FlowFile& ff) override; + void transfer(api::core::FlowFile ff, const minifi::core::Relationship& relationship) override; + void remove(api::core::FlowFile ff) override; + void write(api::core::FlowFile& ff, const io::OutputStreamCallback& callback) override; + void read(api::core::FlowFile& ff, const io::InputStreamCallback& callback) override; + void setAttribute(api::core::FlowFile& ff, std::string_view key, std::string value) override; + void removeAttribute(api::core::FlowFile& ff, std::string_view key) override; + std::optional getAttribute(api::core::FlowFile& ff, std::string_view key) override; + [[nodiscard]] std::map getAttributes(const api::core::FlowFile& ff) const override; + [[nodiscard]] std::string getFlowFileId(const api::core::FlowFile& ff) const override; + [[nodiscard]] uint64_t getFlowFileSize(const api::core::FlowFile& ff) const override; + + void addInputFlowFile(std::unique_ptr flow_file); + + private: + std::vector> input_flow_files_; + std::map>> transferred_flow_files_; + std::vector> removed_flow_files_; +}; + +} // namespace org::apache::nifi::minifi::mock diff --git a/extension-framework/cpp-extension-lib/mocklib/include/MockStreams.h b/extension-framework/cpp-extension-lib/mocklib/include/MockStreams.h new file mode 100644 index 0000000000..b5204c58c6 --- /dev/null +++ b/extension-framework/cpp-extension-lib/mocklib/include/MockStreams.h @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "minifi-cpp/io/InputStream.h" +#include "minifi-cpp/io/OutputStream.h" + +namespace org::apache::nifi::minifi::mock { + +class MockOutputStream : public io::OutputStream { + public: + explicit MockOutputStream(std::vector& container) : container_(container) {} + + size_t write(const uint8_t* value, const size_t len) override { + if (len == 0 || value == nullptr) { return 0; } + + if (offset_ + len > container_.size()) { container_.resize(offset_ + len); } + + std::memcpy(container_.data() + offset_, value, len); + offset_ += len; + return len; + } + + void seek(const size_t offset) override { offset_ = offset; } + + [[nodiscard]] size_t tell() const override { return offset_; } + + [[nodiscard]] std::span getBuffer() const override { return {container_.data(), container_.size()}; } + + int initialize() override { + offset_ = 0; + return 0; + } + + void close() override {} + + private: + std::vector& container_; + size_t offset_ = 0; +}; + +class MockInputStream : public io::InputStream { + public: + explicit MockInputStream(const std::vector& container) : container_(container) {} + + size_t read(std::span out_buffer) override { + if (out_buffer.empty() || offset_ >= container_.size()) { return 0; } + + const size_t readable = container_.size() - offset_; + const size_t len = std::min(out_buffer.size(), readable); + + std::memcpy(out_buffer.data(), container_.data() + offset_, len); + offset_ += len; + return len; + } + + void seek(const size_t offset) override { offset_ = std::min(offset, container_.size()); } + + [[nodiscard]] size_t tell() const override { return offset_; } + + [[nodiscard]] size_t size() const override { return container_.size(); } + + [[nodiscard]] std::span getBuffer() const override { return {container_.data(), container_.size()}; } + + int initialize() override { + offset_ = 0; + return 0; + } + + void close() override {} + + private: + const std::vector& container_; + size_t offset_ = 0; +}; + +} // namespace org::apache::nifi::minifi::mock diff --git a/extension-framework/cpp-extension-lib/mocklib/include/MockUtils.h b/extension-framework/cpp-extension-lib/mocklib/include/MockUtils.h new file mode 100644 index 0000000000..cae06ca4f2 --- /dev/null +++ b/extension-framework/cpp-extension-lib/mocklib/include/MockUtils.h @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "MockLogger.h" +#include "minifi-cpp/core/ProcessorMetadata.h" + +namespace org::apache::nifi::minifi::mock { +inline core::ProcessorMetadata getMockMetadata() { + return core::ProcessorMetadata{.uuid = utils::Identifier{}, .name = "Processor", .logger = std::make_shared()}; +} + +inline core::ProcessorMetadata metadataWithLogger(std::shared_ptr logger) { + return core::ProcessorMetadata{.uuid = utils::Identifier{}, .name = "Processor", .logger = std::move(logger)}; +} +} // namespace org::apache::nifi::minifi::mock diff --git a/extension-framework/cpp-extension-lib/mocklib/src/MockProcessContext.cpp b/extension-framework/cpp-extension-lib/mocklib/src/MockProcessContext.cpp new file mode 100644 index 0000000000..2b571afcfa --- /dev/null +++ b/extension-framework/cpp-extension-lib/mocklib/src/MockProcessContext.cpp @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "MockProcessContext.h" + +#include "utils/PropertyErrors.h" + +namespace org::apache::nifi::minifi::mock { +std::expected MockProcessContext::getProperty(const std::string_view name, const api::core::FlowFile*) const { + if (!properties_.contains(name)) { return std::unexpected{make_error_code(core::PropertyErrorCode::PropertyNotSet)}; } + return properties_.at(std::string(name)); +} + +std::expected MockProcessContext::getProperty(const minifi::core::PropertyReference& property_reference, + const api::core::FlowFile* flow_file) const { + if (auto property = getProperty(property_reference.name, flow_file)) { return property; } + if (property_reference.default_value) { return std::string{*property_reference.default_value}; } + return std::unexpected{make_error_code(core::PropertyErrorCode::PropertyNotSet)}; +} + +std::expected MockProcessContext::getControllerService(const minifi::core::PropertyReference&) const { + return nullptr; +} + +std::map MockProcessContext::getDynamicProperties(const api::core::FlowFile*) const { + return {}; +} + +bool MockProcessContext::hasNonEmptyProperty(const std::string_view name) const { + return properties_.contains(name); +} + +std::expected, std::error_code> MockProcessContext::getSslData(const minifi::core::PropertyReference&) const { + return std::nullopt; +} + +[[nodiscard]] std::expected, std::error_code> MockProcessContext::getProxyData(const minifi::core::PropertyReference&) const { + return std::nullopt; +} +} // namespace org::apache::nifi::minifi::mock diff --git a/extension-framework/cpp-extension-lib/mocklib/src/MockProcessSession.cpp b/extension-framework/cpp-extension-lib/mocklib/src/MockProcessSession.cpp new file mode 100644 index 0000000000..011ba4c934 --- /dev/null +++ b/extension-framework/cpp-extension-lib/mocklib/src/MockProcessSession.cpp @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "MockProcessSession.h" + +#include "MockStreams.h" + +namespace org::apache::nifi::minifi::mock { + +api::core::FlowFile MockProcessSession::create(const api::core::FlowFile*) { + auto new_ff = api::core::FlowFile{new MinifiFlowFile}; // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) + return new_ff; +} +api::core::FlowFile MockProcessSession::get() { + if (input_flow_files_.empty()) { return nullptr; } + + auto ff = std::move(input_flow_files_.back()); + input_flow_files_.pop_back(); + + return api::core::FlowFile{ff.release()}; +} + +void MockProcessSession::penalize(api::core::FlowFile& ff) { + ff->is_penalized = true; +} + +void MockProcessSession::transfer(api::core::FlowFile ff, const minifi::core::Relationship& relationship) { + transferred_flow_files_[relationship.getName()].push_back(std::unique_ptr(ff.release())); +} + +void MockProcessSession::remove(api::core::FlowFile ff) { + removed_flow_files_.push_back(std::unique_ptr(ff.release())); +} + +void MockProcessSession::write(api::core::FlowFile& ff, const io::OutputStreamCallback& callback) { + const auto stream = std::make_shared(ff->content); + callback(stream); +} + +void MockProcessSession::read(api::core::FlowFile& ff, const io::InputStreamCallback& callback) { + const auto stream = std::make_shared(ff->content); + callback(stream); +} + +void MockProcessSession::setAttribute(api::core::FlowFile& ff, std::string_view key, std::string value) { + ff->attributes[std::string(key)] = std::move(value); +} + +void MockProcessSession::removeAttribute(api::core::FlowFile& ff, std::string_view key) { + ff->attributes.erase(std::string(key)); +} + +std::optional MockProcessSession::getAttribute(api::core::FlowFile& ff, std::string_view key) { + auto& attributes = ff->attributes; + if (const auto it = attributes.find(std::string(key)); it != attributes.end()) { return it->second; } + return std::nullopt; +} + +std::map MockProcessSession::getAttributes(const api::core::FlowFile& ff) const { + return ff->attributes; +} + +std::string MockProcessSession::getFlowFileId(const api::core::FlowFile& ff) const { + return ff->id; +} + +uint64_t MockProcessSession::getFlowFileSize(const api::core::FlowFile& ff) const { + return ff->content.size(); +} + +void MockProcessSession::addInputFlowFile(std::unique_ptr flow_file) { + input_flow_files_.push_back(std::move(flow_file)); +} +} // namespace org::apache::nifi::minifi::mock diff --git a/extension-framework/cpp-extension-lib/mocklib/src/mock-minifi-c.cpp b/extension-framework/cpp-extension-lib/mocklib/src/mock-minifi-c.cpp new file mode 100644 index 0000000000..6893ac9599 --- /dev/null +++ b/extension-framework/cpp-extension-lib/mocklib/src/mock-minifi-c.cpp @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "minifi-c.h" + +MinifiExtension* MINIFI_REGISTER_EXTENSION_FN(MinifiExtensionContext*, const MinifiExtensionDefinition*) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiRegisterProcessor(MinifiExtension*, const MinifiProcessorClassDefinition*) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiRegisterControllerService(MinifiExtension*, const MinifiControllerServiceClassDefinition*) { + throw std::runtime_error("Not implemented"); +} + +MINIFI_OWNED MinifiPublishedMetrics* MinifiPublishedMetricsCreate(size_t, const MinifiStringView*, const double*) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiProcessContextGetProperty(MinifiProcessContext*, MinifiStringView, MinifiFlowFile*, + void (*)(void* user_ctx, MinifiStringView property_value), void*) { + throw std::runtime_error("Not implemented"); +} +MinifiBool MinifiProcessContextHasNonEmptyProperty(MinifiProcessContext*, MinifiStringView) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiProcessContextGetControllerServiceFromProperty(MinifiProcessContext*, MinifiStringView, MinifiStringView, MinifiControllerService**) { + throw std::runtime_error("Not implemented"); +} +void MinifiProcessContextGetDynamicProperties(MinifiProcessContext*, MinifiFlowFile*, + void (*)(void* user_ctx, MinifiStringView dynamic_property_name, MinifiStringView dynamic_property_value), void*) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiProcessContextGetSslData(MinifiProcessContext*, MinifiStringView, + void (*)(void* user_ctx, const MinifiSslData* ssl_data), void*) { + throw std::runtime_error("Not implemented"); +} + +void MinifiLoggerSetMaxLogSize(MinifiLogger*, int32_t) { + throw std::runtime_error("Not implemented"); +} +void MinifiLoggerLogString(MinifiLogger*, MinifiLogLevel, MinifiStringView) { + throw std::runtime_error("Not implemented"); +} +MinifiBool MinifiLoggerShouldLog(MinifiLogger*, MinifiLogLevel) { + throw std::runtime_error("Not implemented"); +} +MinifiLogLevel MinifiLoggerLevel(MinifiLogger*) { + throw std::runtime_error("Not implemented"); +} + +MINIFI_OWNED MinifiFlowFile* MinifiProcessSessionGet(MinifiProcessSession*) { + throw std::runtime_error("Not implemented"); +} +MINIFI_OWNED MinifiFlowFile* MinifiProcessSessionCreate(MinifiProcessSession*, MinifiFlowFile*) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiProcessSessionPenalize(MinifiProcessSession*, MinifiFlowFile*) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiProcessSessionTransfer(MinifiProcessSession*, MINIFI_OWNED MinifiFlowFile*, MinifiStringView) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiProcessSessionRemove(MinifiProcessSession*, MINIFI_OWNED MinifiFlowFile*) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiProcessSessionRead(MinifiProcessSession*, MinifiFlowFile*, int64_t (*)(void* user_ctx, MinifiInputStream*), void*) { + throw std::runtime_error("Not implemented"); +} +MinifiStatus MinifiProcessSessionWrite(MinifiProcessSession*, MinifiFlowFile*, int64_t (*)(void* user_ctx, MinifiOutputStream*), void*) { + throw std::runtime_error("Not implemented"); +} + +void MinifiConfigGet(MinifiExtensionContext*, MinifiStringView, void (*)(void* user_ctx, MinifiStringView config_value), void*) { + throw std::runtime_error("Not implemented"); +} + +size_t MinifiInputStreamSize(MinifiInputStream*) { + throw std::runtime_error("Not implemented"); +} + +int64_t MinifiInputStreamRead(MinifiInputStream*, char*, size_t) { + throw std::runtime_error("Not implemented"); +} +int64_t MinifiOutputStreamWrite(MinifiOutputStream*, const char*, size_t) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiProcessSessionSetFlowFileAttribute(MinifiProcessSession*, MinifiFlowFile*, MinifiStringView, const MinifiStringView*) { + throw std::runtime_error("Not implemented"); +} +MinifiBool MinifiProcessSessionGetFlowFileAttribute(MinifiProcessSession*, MinifiFlowFile*, MinifiStringView, + void (*)(void* user_ctx, MinifiStringView attribute_value), void*) { + throw std::runtime_error("Not implemented"); +} +void MinifiProcessSessionGetFlowFileAttributes(MinifiProcessSession*, MinifiFlowFile*, + void (*)(void* user_ctx, MinifiStringView attribute_name, MinifiStringView attribute_value), void*) { + throw std::runtime_error("Not implemented"); +} +uint64_t MinifiProcessSessionGetFlowFileSize(MinifiProcessSession*, MinifiFlowFile*) { + throw std::runtime_error("Not implemented"); +} +MinifiStatus MinifiProcessSessionGetFlowFileId(MinifiProcessSession*, MinifiFlowFile*, void (*)(void* user_ctx, MinifiStringView flow_file_id), + void*) { + throw std::runtime_error("Not implemented"); +} + +MinifiStatus MinifiControllerServiceContextGetProperty(MinifiControllerServiceContext*, MinifiStringView, + void (*)(void* user_ctx, MinifiStringView property_value), void*) { + throw std::runtime_error("Not implemented"); +} diff --git a/extension-framework/cpp-extension-lib/src/core/ControllerServiceContext.cpp b/extension-framework/cpp-extension-lib/src/core/ControllerServiceContext.cpp index 089112c042..be5104c3b8 100644 --- a/extension-framework/cpp-extension-lib/src/core/ControllerServiceContext.cpp +++ b/extension-framework/cpp-extension-lib/src/core/ControllerServiceContext.cpp @@ -22,7 +22,7 @@ namespace org::apache::nifi::minifi::api::core { std::expected ControllerServiceContext::getProperty(const std::string_view name) const { std::optional value = std::nullopt; - const MinifiStatus status = MinifiControllerServiceContextGetProperty(impl_, utils::toStringView(name), + const MinifiStatus status = MinifiControllerServiceContextGetProperty(impl_, utils::minifiStringView(name), [] (void* data, const MinifiStringView result) { (*static_cast*>(data)) = std::string(result.data, result.length); }, &value); diff --git a/extension-framework/cpp-extension-lib/src/core/ProcessContext.cpp b/extension-framework/cpp-extension-lib/src/core/ProcessContext.cpp index d260b9c0c9..9677a7ca34 100644 --- a/extension-framework/cpp-extension-lib/src/core/ProcessContext.cpp +++ b/extension-framework/cpp-extension-lib/src/core/ProcessContext.cpp @@ -1,5 +1,5 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one or more + * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 @@ -16,39 +16,107 @@ */ #include "api/core/ProcessContext.h" -#include "api/utils/minifi-c-utils.h" + #include "api/core/FlowFile.h" +#include "api/utils/minifi-c-utils.h" namespace org::apache::nifi::minifi::api::core { -std::expected ProcessContext::getProperty(std::string_view name, const FlowFile* flow_file) const { +std::expected CffiProcessContext::getProperty(const minifi::core::PropertyReference& property_reference, + const FlowFile* flow_file) const { + return getProperty(property_reference.name, flow_file); +} + +std::expected CffiProcessContext::getProperty(std::string_view name, const FlowFile* flow_file) const { std::optional value; - MinifiStatus status = MinifiProcessContextGetProperty(impl_, utils::toStringView(name), flow_file ? flow_file->get() : MINIFI_NULL, - [] (void* data, MinifiStringView result) { - (*static_cast*>(data)) = std::string(result.data, result.length); - }, &value); + const MinifiStatus status = MinifiProcessContextGetProperty( + impl_, + utils::minifiStringView(name), + flow_file ? flow_file->get() : MINIFI_NULL, + [](void* data, const MinifiStringView result) { (*static_cast*>(data)) = std::string(result.data, result.length); }, + &value); - if (!value) { - return std::unexpected{utils::make_error_code(status)}; - } + if (!value) { return std::unexpected{utils::make_error_code(status)}; } return value.value(); } -bool ProcessContext::hasNonEmptyProperty(std::string_view name) const { - return MinifiProcessContextHasNonEmptyProperty(impl_, utils::toStringView(name)); +bool CffiProcessContext::hasNonEmptyProperty(std::string_view name) const { + return MinifiProcessContextHasNonEmptyProperty(impl_, utils::minifiStringView(name)); } -std::expected ProcessContext::getControllerService(const std::string_view controller_service_name, - const std::string_view controller_service_class) const { +std::expected CffiProcessContext::getControllerService(const minifi::core::PropertyReference& prop) const { MinifiControllerService* controller_service = nullptr; - if (const MinifiStatus status = MinifiProcessContextGetControllerService(impl_, - utils::toStringView(controller_service_name), - utils::toStringView(controller_service_class), + gsl_Assert(prop.allowed_types.size() == 1); + const MinifiStatus status = MinifiProcessContextGetControllerServiceFromProperty(impl_, + utils::minifiStringView(prop.name), + utils::minifiStringView(prop.allowed_types[0]), &controller_service); - status != MINIFI_STATUS_SUCCESS) { + if (status == MINIFI_STATUS_PROPERTY_NOT_SET) { + return nullptr; + } + if (status != MINIFI_STATUS_SUCCESS) { return std::unexpected{utils::make_error_code(status)}; } return controller_service; } +std::map CffiProcessContext::getDynamicProperties(const FlowFile* flow_file) const { + std::map result; + MinifiProcessContextGetDynamicProperties( + impl_, + flow_file ? flow_file->get() : MINIFI_NULL, + [](void* user_ctx, const MinifiStringView key, const MinifiStringView value) { + static_cast*>(user_ctx)->emplace(utils::toString(key), utils::toString(value)); + }, + &result); + return result; +} + +std::expected, std::error_code> CffiProcessContext::getSslData(const minifi::core::PropertyReference& prop) const { + const auto controller_name = getProperty(prop, nullptr); + if (!controller_name) { return std::nullopt; } + + auto ssl_data = utils::net::SslData{}; + + if (const auto status = MinifiProcessContextGetSslData(impl_, utils::minifiStringView(*controller_name), [](void* data, const MinifiSslData* minifi_ssl_data) { + auto* my_ssl_data = static_cast(data); + my_ssl_data->ca_loc = utils::toString(minifi_ssl_data->ca_certificate_file); + my_ssl_data->cert_loc = utils::toString(minifi_ssl_data->certificate_file); + my_ssl_data->key_loc = utils::toString(minifi_ssl_data->private_key_file); + my_ssl_data->key_pw = utils::toString(minifi_ssl_data->passphrase); + }, &ssl_data); + status != MINIFI_STATUS_SUCCESS) { + return std::unexpected{utils::make_error_code(status)}; + } + + return ssl_data; +} + +std::expected, std::error_code> CffiProcessContext::getProxyData(const minifi::core::PropertyReference& prop) const { + auto proxy_data = utils::ProxyData{}; + const auto status = MinifiProcessContextGetProxyDataFromProperty( + impl_, + utils::minifiStringView(prop.name), + [](void* data, const MinifiProxyData* minifi_proxy_data) { + auto* proxy = static_cast(data); + proxy->host = utils::toString(minifi_proxy_data->hostname); + proxy->port = minifi_proxy_data->port; + if (minifi_proxy_data->password && minifi_proxy_data->username) { + proxy->proxy_credentials = utils::BasicAuthCredentials{.username = utils::toString(*minifi_proxy_data->username), + .password = utils::toString(*minifi_proxy_data->password)}; + } else { + proxy->proxy_credentials = std::nullopt; + } + if (minifi_proxy_data->proxy_type == MINIFI_PROXY_TYPE_HTTP) { + proxy->proxy_type = utils::ProxyType::HTTP; + } else { + proxy->proxy_type = utils::ProxyType::DIRECT; + } + }, + &proxy_data); + if (status == MINIFI_STATUS_PROPERTY_NOT_SET) { return std::nullopt; } + if (status == MINIFI_STATUS_SUCCESS) { return proxy_data; } + return std::unexpected{utils::make_error_code(status)}; +} + } // namespace org::apache::nifi::minifi::api::core diff --git a/extension-framework/cpp-extension-lib/src/core/ProcessSession.cpp b/extension-framework/cpp-extension-lib/src/core/ProcessSession.cpp index 8b13b09c2a..5bf12b1d64 100644 --- a/extension-framework/cpp-extension-lib/src/core/ProcessSession.cpp +++ b/extension-framework/cpp-extension-lib/src/core/ProcessSession.cpp @@ -69,28 +69,34 @@ class MinifiInputStreamWrapper : public io::InputStreamImpl { } // namespace -FlowFile ProcessSession::get() { +FlowFile CffiProcessSession::get() { return FlowFile{MinifiProcessSessionGet(impl_)}; } -FlowFile ProcessSession::create(const FlowFile* parent) { +FlowFile CffiProcessSession::create(const FlowFile* parent) { return FlowFile{MinifiProcessSessionCreate(impl_, parent ? parent->get() : MINIFI_NULL)}; } -void ProcessSession::transfer(FlowFile ff, const minifi::core::Relationship& relationship) { +void CffiProcessSession::penalize(FlowFile& ff) { + if (MINIFI_STATUS_SUCCESS != MinifiProcessSessionPenalize(impl_, ff.get())) { + throw minifi::Exception(minifi::FILE_OPERATION_EXCEPTION, "Failed to penalize flowfile"); + } +} + +void CffiProcessSession::transfer(FlowFile ff, const minifi::core::Relationship& relationship) { const auto rel_name = relationship.getName(); - if (MINIFI_STATUS_SUCCESS != MinifiProcessSessionTransfer(impl_, ff.release(), utils::toStringView(rel_name))) { + if (MINIFI_STATUS_SUCCESS != MinifiProcessSessionTransfer(impl_, ff.release(), utils::minifiStringView(rel_name))) { throw minifi::Exception(minifi::FILE_OPERATION_EXCEPTION, "Failed to transfer flowfile"); } } -void ProcessSession::remove(FlowFile ff) { +void CffiProcessSession::remove(FlowFile ff) { if (MINIFI_STATUS_SUCCESS != MinifiProcessSessionRemove(impl_, ff.release())) { throw minifi::Exception(minifi::FILE_OPERATION_EXCEPTION, "Failed to remove flowfile"); } } -void ProcessSession::write(FlowFile& flow_file, const io::OutputStreamCallback& callback) { +void CffiProcessSession::write(FlowFile& flow_file, const io::OutputStreamCallback& callback) { const auto status = MinifiProcessSessionWrite( impl_, flow_file.get(), @@ -103,7 +109,7 @@ void ProcessSession::write(FlowFile& flow_file, const io::OutputStreamCallback& if (status != MINIFI_STATUS_SUCCESS) { throw minifi::Exception(minifi::FILE_OPERATION_EXCEPTION, "Failed to process flowfile content"); } } -void ProcessSession::read(FlowFile& flow_file, const io::InputStreamCallback& callback) { +void CffiProcessSession::read(FlowFile& flow_file, const io::InputStreamCallback& callback) { const auto status = MinifiProcessSessionRead( impl_, flow_file.get(), @@ -116,28 +122,28 @@ void ProcessSession::read(FlowFile& flow_file, const io::InputStreamCallback& ca if (status != MINIFI_STATUS_SUCCESS) { throw minifi::Exception(minifi::FILE_OPERATION_EXCEPTION, "Failed to process flowfile content"); } } -void ProcessSession::setAttribute(FlowFile& ff, const std::string_view key, std::string value) { // NOLINT(performance-unnecessary-value-param) - const MinifiStringView value_ref = utils::toStringView(value); - if (MINIFI_STATUS_SUCCESS != MinifiProcessSessionSetFlowFileAttribute(impl_, ff.get(), utils::toStringView(key), &value_ref)) { +void CffiProcessSession::setAttribute(FlowFile& ff, const std::string_view key, std::string value) { // NOLINT(performance-unnecessary-value-param) + const MinifiStringView value_ref = utils::minifiStringView(value); + if (MINIFI_STATUS_SUCCESS != MinifiProcessSessionSetFlowFileAttribute(impl_, ff.get(), utils::minifiStringView(key), &value_ref)) { throw minifi::Exception(minifi::FILE_OPERATION_EXCEPTION, "Failed to set attribute"); } } -void ProcessSession::removeAttribute(FlowFile& ff, const std::string_view key) { - if (MINIFI_STATUS_SUCCESS != MinifiProcessSessionSetFlowFileAttribute(impl_, ff.get(), utils::toStringView(key), nullptr)) { +void CffiProcessSession::removeAttribute(FlowFile& ff, const std::string_view key) { + if (MINIFI_STATUS_SUCCESS != MinifiProcessSessionSetFlowFileAttribute(impl_, ff.get(), utils::minifiStringView(key), nullptr)) { throw minifi::Exception(minifi::FILE_OPERATION_EXCEPTION, "Failed to remove attribute"); } } -std::optional ProcessSession::getAttribute(FlowFile& ff, std::string_view key) { +std::optional CffiProcessSession::getAttribute(FlowFile& ff, std::string_view key) { std::optional result; - MinifiProcessSessionGetFlowFileAttribute(impl_, ff.get(), utils::toStringView(key), [] (void* user_ctx, MinifiStringView value) { + MinifiProcessSessionGetFlowFileAttribute(impl_, ff.get(), utils::minifiStringView(key), [] (void* user_ctx, MinifiStringView value) { *static_cast*>(user_ctx) = std::string{value.data, value.length}; }, &result); return result; } -std::map ProcessSession::getAttributes(FlowFile& ff) { +std::map CffiProcessSession::getAttributes(const FlowFile& ff) const { std::map result; MinifiProcessSessionGetFlowFileAttributes(impl_, ff.get(), [] (void* user_ctx, const MinifiStringView key, const MinifiStringView value) { static_cast*>(user_ctx)->insert({std::string{key.data, key.length}, std::string{value.data, value.length}}); @@ -145,6 +151,18 @@ std::map ProcessSession::getAttributes(FlowFile& ff) { return result; } +std::string CffiProcessSession::getFlowFileId(const FlowFile& ff) const { + std::string result; + MinifiProcessSessionGetFlowFileId(impl_, ff.get(), [](void* user_ctx, const MinifiStringView value) { + *static_cast(user_ctx) = std::string{value.data, value.length}; + }, &result); + return result; +} + +uint64_t CffiProcessSession::getFlowFileSize(const FlowFile& ff) const { + return MinifiProcessSessionGetFlowFileSize(impl_, ff.get()); +} + void ProcessSession::writeBuffer(FlowFile& flow_file, std::span buffer) { writeBuffer(flow_file, as_bytes(buffer)); } diff --git a/extension-framework/cpp-extension-lib/src/core/logging/Logger.cpp b/extension-framework/cpp-extension-lib/src/core/logging/Logger.cpp index d91a67f609..4f5f2f422e 100644 --- a/extension-framework/cpp-extension-lib/src/core/logging/Logger.cpp +++ b/extension-framework/cpp-extension-lib/src/core/logging/Logger.cpp @@ -49,19 +49,19 @@ minifi::core::logging::LOG_LEVEL toLogLevel(MinifiLogLevel level) { } // namespace -void Logger::set_max_log_size(int size) { +void CffiLogger::set_max_log_size(const int size) { MinifiLoggerSetMaxLogSize(impl_, size); } -void Logger::log_string(minifi::core::logging::LOG_LEVEL level, std::string str) { +void CffiLogger::log_string(const minifi::core::logging::LOG_LEVEL level, const std::string str) { MinifiLoggerLogString(impl_, toCLogLevel(level), MinifiStringView{.data = str.data(), .length = str.length()}); } -bool Logger::should_log(minifi::core::logging::LOG_LEVEL level) { +bool CffiLogger::should_log(const minifi::core::logging::LOG_LEVEL level) { return MinifiLoggerShouldLog(impl_, toCLogLevel(level)); } -[[nodiscard]] minifi::core::logging::LOG_LEVEL Logger::level() const { +[[nodiscard]] minifi::core::logging::LOG_LEVEL CffiLogger::level() const { return toLogLevel(MinifiLoggerLevel(impl_)); } diff --git a/extensions/gcp/CMakeLists.txt b/extensions/gcp/CMakeLists.txt index 654209d080..2fc42260c2 100644 --- a/extensions/gcp/CMakeLists.txt +++ b/extensions/gcp/CMakeLists.txt @@ -30,8 +30,8 @@ add_minifi_library(minifi-gcp SHARED ${SOURCES}) if (NOT WIN32) target_compile_options(minifi-gcp PRIVATE -Wno-error=deprecated-declarations) # Suppress deprecation warnings for std::rel_ops usage endif() -target_link_libraries(minifi-gcp ${LIBMINIFI} google-cloud-cpp::storage) -target_include_directories(minifi-gcp SYSTEM PUBLIC ${google-cloud-cpp_INCLUDE_DIRS}) +target_link_libraries(minifi-gcp minifi-cpp-extension-lib google-cloud-cpp::storage) -register_extension(minifi-gcp "GCP EXTENSIONS" GCP-EXTENSIONS "This enables Google Cloud Platform support" "extensions/gcp/tests") +target_include_directories(minifi-gcp SYSTEM PUBLIC ${google-cloud-cpp_INCLUDE_DIRS}) +register_c_api_extension(minifi-gcp "GCP EXTENSIONS" GCP-EXTENSIONS "This enables Google Cloud Platform support" "extensions/gcp/tests") diff --git a/extensions/gcp/ExtensionInitializer.cpp b/extensions/gcp/ExtensionInitializer.cpp new file mode 100644 index 0000000000..89b00e9ff5 --- /dev/null +++ b/extensions/gcp/ExtensionInitializer.cpp @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../../extension-framework/cpp-extension-lib/include/api/core/Resource.h" +#include "api/core/Resource.h" +#include "api/utils/minifi-c-utils.h" +#include "processors/DeleteGCSObject.h" +#include "processors/FetchGCSObject.h" +#include "processors/ListGCSBucket.h" +#include "processors/PutGCSObject.h" + +#define MKSOC(x) #x +#define MAKESTRING(x) MKSOC(x) // NOLINT(cppcoreguidelines-macro-usage) + +namespace minifi = org::apache::nifi::minifi; + +CEXTENSIONAPI const uint32_t MinifiApiVersion = MINIFI_API_VERSION; + +CEXTENSIONAPI void MinifiInitExtension(MinifiExtensionContext* extension_context) { + const MinifiExtensionDefinition extension_definition{ + .name = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_NAME)), + .version = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_VERSION)), + .deinit = nullptr, + .user_data = nullptr + }; + auto* extension = MinifiRegisterExtension(extension_context, &extension_definition); + minifi::api::core::registerProcessors(extension); + minifi::api::core::registerControllerServices(extension); +} diff --git a/extensions/gcp/GCPAttributes.h b/extensions/gcp/GCPAttributes.h index 5657e72005..0ec973ea60 100644 --- a/extensions/gcp/GCPAttributes.h +++ b/extensions/gcp/GCPAttributes.h @@ -19,8 +19,9 @@ #include +#include "api/core/FlowFile.h" +#include "api/core/ProcessSession.h" #include "google/cloud/storage/object_metadata.h" -#include "minifi-cpp/core/FlowFile.h" namespace org::apache::nifi::minifi::extensions::gcp { @@ -50,32 +51,32 @@ constexpr std::string_view GCS_SELF_LINK_ATTR = "gcs.self.link"; constexpr std::string_view GCS_ENCRYPTION_ALGORITHM_ATTR = "gcs.encryption.algorithm"; constexpr std::string_view GCS_ENCRYPTION_SHA256_ATTR = "gcs.encryption.sha256"; -inline void setAttributesFromObjectMetadata(core::FlowFile& flow_file, const ::google::cloud::storage::ObjectMetadata& object_metadata) { - flow_file.setAttribute(GCS_BUCKET_ATTR, object_metadata.bucket()); - flow_file.setAttribute(GCS_OBJECT_NAME_ATTR, object_metadata.name()); - flow_file.setAttribute(GCS_SIZE_ATTR, std::to_string(object_metadata.size())); - flow_file.setAttribute(GCS_CRC32C_ATTR, object_metadata.crc32c()); - flow_file.setAttribute(GCS_MD5_ATTR, object_metadata.md5_hash()); - flow_file.setAttribute(GCS_CONTENT_ENCODING_ATTR, object_metadata.content_encoding()); - flow_file.setAttribute(GCS_CONTENT_LANGUAGE_ATTR, object_metadata.content_language()); - flow_file.setAttribute(GCS_CONTENT_DISPOSITION_ATTR, object_metadata.content_disposition()); - flow_file.setAttribute(GCS_CREATE_TIME_ATTR, std::to_string(std::chrono::duration_cast(object_metadata.time_created().time_since_epoch()).count())); - flow_file.setAttribute(GCS_UPDATE_TIME_ATTR, std::to_string(std::chrono::duration_cast(object_metadata.updated().time_since_epoch()).count())); - flow_file.setAttribute(GCS_DELETE_TIME_ATTR, std::to_string(std::chrono::duration_cast(object_metadata.time_deleted().time_since_epoch()).count())); - flow_file.setAttribute(GCS_MEDIA_LINK_ATTR, object_metadata.media_link()); - flow_file.setAttribute(GCS_SELF_LINK_ATTR, object_metadata.self_link()); - flow_file.setAttribute(GCS_ETAG_ATTR, object_metadata.etag()); - flow_file.setAttribute(GCS_GENERATED_ID, object_metadata.id()); - flow_file.setAttribute(GCS_META_GENERATION, std::to_string(object_metadata.metageneration())); - flow_file.setAttribute(GCS_GENERATION, std::to_string(object_metadata.generation())); - flow_file.setAttribute(GCS_STORAGE_CLASS, object_metadata.storage_class()); +inline void setAttributesFromObjectMetadata(api::core::FlowFile& flow_file, const ::google::cloud::storage::ObjectMetadata& object_metadata, api::core::ProcessSession& session) { + session.setAttribute(flow_file, GCS_BUCKET_ATTR, object_metadata.bucket()); + session.setAttribute(flow_file, GCS_OBJECT_NAME_ATTR, object_metadata.name()); + session.setAttribute(flow_file, GCS_SIZE_ATTR, std::to_string(object_metadata.size())); + session.setAttribute(flow_file, GCS_CRC32C_ATTR, object_metadata.crc32c()); + session.setAttribute(flow_file, GCS_MD5_ATTR, object_metadata.md5_hash()); + session.setAttribute(flow_file, GCS_CONTENT_ENCODING_ATTR, object_metadata.content_encoding()); + session.setAttribute(flow_file, GCS_CONTENT_LANGUAGE_ATTR, object_metadata.content_language()); + session.setAttribute(flow_file, GCS_CONTENT_DISPOSITION_ATTR, object_metadata.content_disposition()); + session.setAttribute(flow_file, GCS_CREATE_TIME_ATTR, std::to_string(std::chrono::duration_cast(object_metadata.time_created().time_since_epoch()).count())); + session.setAttribute(flow_file, GCS_UPDATE_TIME_ATTR, std::to_string(std::chrono::duration_cast(object_metadata.updated().time_since_epoch()).count())); + session.setAttribute(flow_file, GCS_DELETE_TIME_ATTR, std::to_string(std::chrono::duration_cast(object_metadata.time_deleted().time_since_epoch()).count())); + session.setAttribute(flow_file, GCS_MEDIA_LINK_ATTR, object_metadata.media_link()); + session.setAttribute(flow_file, GCS_SELF_LINK_ATTR, object_metadata.self_link()); + session.setAttribute(flow_file, GCS_ETAG_ATTR, object_metadata.etag()); + session.setAttribute(flow_file, GCS_GENERATED_ID, object_metadata.id()); + session.setAttribute(flow_file, GCS_META_GENERATION, std::to_string(object_metadata.metageneration())); + session.setAttribute(flow_file, GCS_GENERATION, std::to_string(object_metadata.generation())); + session.setAttribute(flow_file, GCS_STORAGE_CLASS, object_metadata.storage_class()); if (object_metadata.has_customer_encryption()) { - flow_file.setAttribute(GCS_ENCRYPTION_ALGORITHM_ATTR, object_metadata.customer_encryption().encryption_algorithm); - flow_file.setAttribute(GCS_ENCRYPTION_SHA256_ATTR, object_metadata.customer_encryption().key_sha256); + session.setAttribute(flow_file, GCS_ENCRYPTION_ALGORITHM_ATTR, object_metadata.customer_encryption().encryption_algorithm); + session.setAttribute(flow_file, GCS_ENCRYPTION_SHA256_ATTR, object_metadata.customer_encryption().key_sha256); } if (object_metadata.has_owner()) { - flow_file.setAttribute(GCS_OWNER_ENTITY_ATTR, object_metadata.owner().entity); - flow_file.setAttribute(GCS_OWNER_ENTITY_ID_ATTR, object_metadata.owner().entity_id); + session.setAttribute(flow_file, GCS_OWNER_ENTITY_ATTR, object_metadata.owner().entity); + session.setAttribute(flow_file, GCS_OWNER_ENTITY_ID_ATTR, object_metadata.owner().entity_id); } } diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp index 9e93c29422..d84bbeb4bd 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp @@ -18,34 +18,36 @@ #include "GCPCredentialsControllerService.h" -#include "core/Resource.h" #include "google/cloud/storage/client.h" -#include "utils/ProcessorConfigUtils.h" -#include "utils/file/FileUtils.h" namespace org::apache::nifi::minifi::extensions::gcp { -void GCPCredentialsControllerService::initialize() { - setSupportedProperties(Properties); +namespace { +// TODO(MINIFICPP-2763) use utils::file::get_content instead +std::string get_content(const std::filesystem::path& file_name) { + std::ifstream file(file_name, std::ifstream::binary); + std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + return content; +} } -std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonPath() const { - const auto json_path = getProperty(JsonFilePath.name); +std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonPath(api::core::ControllerServiceContext& ctx) const { + const auto json_path = ctx.getProperty(JsonFilePath.name); if (!json_path) { logger_->log_error("Missing or invalid {}", JsonFilePath.name); return nullptr; } - if (!utils::file::exists(*json_path)) { + if (std::error_code ec; !std::filesystem::exists(*json_path, ec) || ec) { logger_->log_error("JSON file for GCP credentials '{}' does not exist", *json_path); return nullptr; } - return google::cloud::MakeServiceAccountCredentials(utils::file::get_content(*json_path)); + return google::cloud::MakeServiceAccountCredentials(get_content(*json_path)); } -std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonContents() const { - auto json_contents = getProperty(JsonContents.name); +std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonContents(api::core::ControllerServiceContext& ctx) const { + auto json_contents = ctx.getProperty(JsonContents.name); if (!json_contents) { logger_->log_error("Missing or invalid {}", JsonContents.name); return nullptr; @@ -54,9 +56,9 @@ std::shared_ptr GCPCredentialsControllerService::cre return google::cloud::MakeServiceAccountCredentials(*json_contents); } -void GCPCredentialsControllerService::onEnable() { +MinifiStatus GCPCredentialsControllerService::enableImpl(api::core::ControllerServiceContext& ctx) { std::optional credentials_location; - if (const auto value = getProperty(CredentialsLoc.name)) { + if (const auto value = ctx.getProperty(CredentialsLoc.name)) { credentials_location = magic_enum::enum_cast(*value); } if (!credentials_location) { @@ -68,15 +70,15 @@ void GCPCredentialsControllerService::onEnable() { } else if (*credentials_location == CredentialsLocation::USE_COMPUTE_ENGINE_CREDENTIALS) { credentials_ = google::cloud::MakeComputeEngineCredentials(); } else if (*credentials_location == CredentialsLocation::USE_JSON_FILE) { - credentials_ = createCredentialsFromJsonPath(); + credentials_ = createCredentialsFromJsonPath(ctx); } else if (*credentials_location == CredentialsLocation::USE_JSON_CONTENTS) { - credentials_ = createCredentialsFromJsonContents(); + credentials_ = createCredentialsFromJsonContents(ctx); } else if (*credentials_location == CredentialsLocation::USE_ANONYMOUS_CREDENTIALS) { credentials_ = google::cloud::MakeInsecureCredentials(); } if (!credentials_) logger_->log_error("Couldn't create valid credentials"); + return MINIFI_STATUS_SUCCESS; } -REGISTER_RESOURCE(GCPCredentialsControllerService, ControllerService); } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h index 0eb8fbf8a1..b53971ae20 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h @@ -18,17 +18,15 @@ #pragma once #include -#include #include +#include -#include "core/controller/ControllerServiceBase.h" -#include "minifi-cpp/core/logging/Logger.h" -#include "core/logging/LoggerFactory.h" -#include "minifi-cpp/core/PropertyDefinition.h" +#include "api/core/ControllerServiceImpl.h" +#include "api/utils/Export.h" #include "core/PropertyDefinitionBuilder.h" -#include "utils/Enum.h" - #include "google/cloud/credentials.h" +#include "minifi-cpp/core/PropertyDefinition.h" +#include "utils/Enum.h" namespace org::apache::nifi::minifi::extensions::gcp { enum class CredentialsLocation { @@ -63,7 +61,7 @@ constexpr customize_t enum_name(CredentialsLocation value) namespace org::apache::nifi::minifi::extensions::gcp { -class GCPCredentialsControllerService : public core::controller::ControllerServiceBase, public core::controller::ControllerServiceHandle { +class GCPCredentialsControllerService : public api::core::ControllerServiceImpl { public: EXTENSIONAPI static constexpr const char* Description = "Manages the credentials for Google Cloud Platform. This allows for multiple Google Cloud Platform related processors " "to reference this single controller service so that Google Cloud Platform credentials can be managed and controlled in a central location."; @@ -91,21 +89,16 @@ class GCPCredentialsControllerService : public core::controller::ControllerServi EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; - ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_CONTROLLER_SERVICES - - using ControllerServiceBase::ControllerServiceBase; - - void initialize() override; - void onEnable() override; + using ControllerServiceImpl::ControllerServiceImpl; - [[nodiscard]] ControllerServiceHandle* getControllerServiceHandle() override {return this;} + MinifiStatus enableImpl(api::core::ControllerServiceContext& ctx) override; [[nodiscard]] const auto& getCredentials() const { return credentials_; } protected: - [[nodiscard]] std::shared_ptr createCredentialsFromJsonPath() const; - [[nodiscard]] std::shared_ptr createCredentialsFromJsonContents() const; + [[nodiscard]] std::shared_ptr createCredentialsFromJsonPath(api::core::ControllerServiceContext& ctx) const; + [[nodiscard]] std::shared_ptr createCredentialsFromJsonContents(api::core::ControllerServiceContext& ctx) const; std::shared_ptr credentials_; diff --git a/extensions/gcp/processors/DeleteGCSObject.cpp b/extensions/gcp/processors/DeleteGCSObject.cpp index 4ad4b43271..a7a6d923bd 100644 --- a/extensions/gcp/processors/DeleteGCSObject.cpp +++ b/extensions/gcp/processors/DeleteGCSObject.cpp @@ -17,69 +17,62 @@ #include "DeleteGCSObject.h" -#include "utils/ProcessorConfigUtils.h" #include "../GCPAttributes.h" -#include "minifi-cpp/core/FlowFile.h" -#include "minifi-cpp/core/ProcessContext.h" -#include "core/ProcessSession.h" -#include "core/Resource.h" +#include "api/core/ProcessContext.h" +#include "api/core/ProcessSession.h" +#include "api/core/Resource.h" +#include "api/utils/ProcessorConfigUtils.h" namespace gcs = ::google::cloud::storage; namespace org::apache::nifi::minifi::extensions::gcp { -void DeleteGCSObject::initialize() { - setSupportedProperties(Properties); - setSupportedRelationships(Relationships); -} -void DeleteGCSObject::onTrigger(core::ProcessContext& context, core::ProcessSession& session) { +MinifiStatus DeleteGCSObject::onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) { gsl_Expects(gcp_credentials_); auto flow_file = session.get(); if (!flow_file) { - context.yield(); - return; + return MINIFI_STATUS_PROCESSOR_YIELD; } - auto bucket = context.getProperty(Bucket, flow_file.get()); + auto bucket = api::utils::parseOptionalProperty(context, Bucket, &flow_file); if (!bucket || bucket->empty()) { logger_->log_error("Missing bucket name"); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } - auto object_name = context.getProperty(Key, flow_file.get()); + auto object_name = api::utils::parseOptionalProperty(context, Key, &flow_file); if (!object_name || object_name->empty()) { logger_->log_error("Missing object name"); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } gcs::Generation generation; - if (const auto object_generation_str = context.getProperty(ObjectGeneration, flow_file.get()); object_generation_str && !object_generation_str->empty()) { + if (auto object_generation_str = api::utils::parseOptionalProperty(context, ObjectGeneration, &flow_file); object_generation_str && !object_generation_str->empty()) { if (const auto geni64 = parsing::parseIntegral(*object_generation_str)) { generation = gcs::Generation{*geni64}; } else { logger_->log_error("Invalid generation: {}", *object_generation_str); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } } auto status = getClient().DeleteObject(*bucket, *object_name, generation, gcs::IfGenerationNotMatch(0)); if (!status.ok()) { - flow_file->setAttribute(GCS_STATUS_MESSAGE, status.message()); - flow_file->setAttribute(GCS_ERROR_REASON, status.error_info().reason()); - flow_file->setAttribute(GCS_ERROR_DOMAIN, status.error_info().domain()); + session.setAttribute(flow_file, GCS_STATUS_MESSAGE, status.message()); + session.setAttribute(flow_file, GCS_ERROR_REASON, status.error_info().reason()); + session.setAttribute(flow_file, GCS_ERROR_DOMAIN, status.error_info().domain()); logger_->log_error("Failed to delete {} object from {} bucket on Google Cloud Storage {} {}", *object_name, *bucket, status.message(), status.error_info().reason()); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } - session.transfer(flow_file, Success); + session.transfer(std::move(flow_file), Success); + return MINIFI_STATUS_SUCCESS; } -REGISTER_RESOURCE(DeleteGCSObject, Processor); - } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/DeleteGCSObject.h b/extensions/gcp/processors/DeleteGCSObject.h index a5d0228476..48364cd45e 100644 --- a/extensions/gcp/processors/DeleteGCSObject.h +++ b/extensions/gcp/processors/DeleteGCSObject.h @@ -17,24 +17,20 @@ #pragma once -#include -#include -#include - #include "../GCPAttributes.h" #include "GCSProcessor.h" -#include "core/logging/LoggerFactory.h" #include "minifi-cpp/core/OutputAttributeDefinition.h" #include "minifi-cpp/core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "utils/ArrayUtils.h" +#include "minifi-cpp/core/Annotation.h" + namespace org::apache::nifi::minifi::extensions::gcp { class DeleteGCSObject : public GCSProcessor { public: using GCSProcessor::GCSProcessor; - ~DeleteGCSObject() override = default; EXTENSIONAPI static constexpr const char* Description = "Deletes an object from a Google Cloud Bucket."; @@ -79,10 +75,8 @@ class DeleteGCSObject : public GCSProcessor { EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; EXTENSIONAPI static constexpr bool IsSingleThreaded = false; - ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_PROCESSORS - - void initialize() override; - void onTrigger(core::ProcessContext& context, core::ProcessSession& session) override; + protected: + MinifiStatus onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) override; }; } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/FetchGCSObject.cpp b/extensions/gcp/processors/FetchGCSObject.cpp index e43b80415a..66614a8a22 100644 --- a/extensions/gcp/processors/FetchGCSObject.cpp +++ b/extensions/gcp/processors/FetchGCSObject.cpp @@ -19,11 +19,9 @@ #include -#include "core/Resource.h" -#include "minifi-cpp/core/FlowFile.h" -#include "minifi-cpp/core/ProcessContext.h" -#include "core/ProcessSession.h" #include "../GCPAttributes.h" +#include "api/utils/ProcessorConfigUtils.h" +#include "minifi-cpp/io/OutputStream.h" namespace gcs = ::google::cloud::storage; @@ -82,80 +80,78 @@ class FetchFromGCSCallback { }; } // namespace - -void FetchGCSObject::initialize() { - setSupportedProperties(Properties); - setSupportedRelationships(Relationships); -} - -void FetchGCSObject::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { - GCSProcessor::onSchedule(context, session_factory); - if (auto encryption_key = context.getProperty(EncryptionKey)) { +MinifiStatus FetchGCSObject::onScheduleImpl(api::core::ProcessContext& context) { + const auto status = GCSProcessor::onScheduleImpl(context); + if (MINIFI_STATUS_SUCCESS != status) { + return status; + } + if (auto encryption_key = context.getProperty(EncryptionKey, nullptr)) { try { encryption_key_ = gcs::EncryptionKey::FromBase64Key(*encryption_key); } catch (const google::cloud::RuntimeStatusError&) { - throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Could not decode the base64-encoded encryption key from property " + std::string(EncryptionKey.name)); } + logger_->log_error("Could not decode the base64-encoded encryption key from property {}", std::string(EncryptionKey.name)); + return MINIFI_STATUS_UNKNOWN_ERROR; + } } + return MINIFI_STATUS_SUCCESS; } -void FetchGCSObject::onTrigger(core::ProcessContext& context, core::ProcessSession& session) { +MinifiStatus FetchGCSObject::onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) { gsl_Expects(gcp_credentials_); auto flow_file = session.get(); if (!flow_file) { - context.yield(); - return; + return MINIFI_STATUS_PROCESSOR_YIELD; } - auto bucket = context.getProperty(Bucket, flow_file.get()); + auto bucket = api::utils::parseOptionalProperty(context, Bucket, &flow_file); if (!bucket || bucket->empty()) { logger_->log_error("Missing bucket name"); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } - auto object_name = context.getProperty(Key, flow_file.get()); + auto object_name = api::utils::parseOptionalProperty(context, Key, &flow_file); if (!object_name || object_name->empty()) { logger_->log_error("Missing object name"); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } gcs::Client client = getClient(); FetchFromGCSCallback callback(client, *bucket, *object_name); callback.setEncryptionKey(encryption_key_); - if (const auto object_generation_str = context.getProperty(ObjectGeneration, flow_file.get()); object_generation_str && !object_generation_str->empty()) { + if (const auto object_generation_str = api::utils::parseOptionalProperty(context, ObjectGeneration, &flow_file); object_generation_str && !object_generation_str->empty()) { if (const auto geni64 = parsing::parseIntegral(*object_generation_str)) { gcs::Generation generation = gcs::Generation{*geni64}; callback.setGeneration(generation); } else { logger_->log_error("Invalid generation: {}", *object_generation_str); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } } session.write(flow_file, std::ref(callback)); if (!callback.getStatus().ok()) { - flow_file->setAttribute(GCS_STATUS_MESSAGE, callback.getStatus().message()); - flow_file->setAttribute(GCS_ERROR_REASON, callback.getStatus().error_info().reason()); - flow_file->setAttribute(GCS_ERROR_DOMAIN, callback.getStatus().error_info().domain()); + session.setAttribute(flow_file, GCS_STATUS_MESSAGE, callback.getStatus().message()); + session.setAttribute(flow_file, GCS_ERROR_REASON, callback.getStatus().error_info().reason()); + session.setAttribute(flow_file, GCS_ERROR_DOMAIN, callback.getStatus().error_info().domain()); logger_->log_error("Failed to fetch from Google Cloud Storage {} {}", callback.getStatus().message(), callback.getStatus().error_info().reason()); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } if (auto generation = callback.getGeneration()) - flow_file->setAttribute(GCS_GENERATION, std::to_string(*generation)); + session.setAttribute(flow_file, GCS_GENERATION, std::to_string(*generation)); if (auto meta_generation = callback.getMetaGeneration()) - flow_file->setAttribute(GCS_META_GENERATION, std::to_string(*meta_generation)); + session.setAttribute(flow_file, GCS_META_GENERATION, std::to_string(*meta_generation)); if (auto storage_class = callback.getStorageClass()) - flow_file->setAttribute(GCS_STORAGE_CLASS, *storage_class); - session.transfer(flow_file, Success); + session.setAttribute(flow_file, GCS_STORAGE_CLASS, *storage_class); + session.transfer(std::move(flow_file), Success); + return MINIFI_STATUS_SUCCESS; } -REGISTER_RESOURCE(FetchGCSObject, Processor); - } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/FetchGCSObject.h b/extensions/gcp/processors/FetchGCSObject.h index 694ad1f5e0..51d72dab98 100644 --- a/extensions/gcp/processors/FetchGCSObject.h +++ b/extensions/gcp/processors/FetchGCSObject.h @@ -17,24 +17,20 @@ #pragma once -#include -#include -#include - #include "../GCPAttributes.h" #include "GCSProcessor.h" #include "minifi-cpp/core/PropertyDefinition.h" #include "minifi-cpp/core/RelationshipDefinition.h" #include "google/cloud/storage/well_known_headers.h" -#include "core/logging/LoggerFactory.h" #include "utils/ArrayUtils.h" +#include "minifi-cpp/core/Annotation.h" + namespace org::apache::nifi::minifi::extensions::gcp { class FetchGCSObject : public GCSProcessor { public: using GCSProcessor::GCSProcessor; - ~FetchGCSObject() override = default; EXTENSIONAPI static constexpr const char* Description = "Fetches a file from a Google Cloud Bucket. Designed to be used in tandem with ListGCSBucket."; @@ -80,11 +76,9 @@ class FetchGCSObject : public GCSProcessor { EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; EXTENSIONAPI static constexpr bool IsSingleThreaded = false; - ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_PROCESSORS - - void initialize() override; - void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override; - void onTrigger(core::ProcessContext& context, core::ProcessSession& session) override; + protected: + MinifiStatus onScheduleImpl(api::core::ProcessContext& context) override; + MinifiStatus onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) override; private: google::cloud::storage::EncryptionKey encryption_key_; diff --git a/extensions/gcp/processors/GCSProcessor.cpp b/extensions/gcp/processors/GCSProcessor.cpp index 274b3f93c0..465782b798 100644 --- a/extensions/gcp/processors/GCSProcessor.cpp +++ b/extensions/gcp/processors/GCSProcessor.cpp @@ -17,45 +17,43 @@ #include "GCSProcessor.h" -#include "utils/ProcessorConfigUtils.h" - #include "../controllerservices/GCPCredentialsControllerService.h" -#include "minifi-cpp/core/ProcessContext.h" -#include "core/ProcessSession.h" +#include "api/utils/ProcessorConfigUtils.h" namespace gcs = ::google::cloud::storage; namespace org::apache::nifi::minifi::extensions::gcp { -std::shared_ptr GCSProcessor::getCredentials(core::ProcessContext& context) const { - auto gcp_credentials_controller_service = utils::parseOptionalControllerService(context, GCSProcessor::GCPCredentials, getUUID()); - if (gcp_credentials_controller_service) { +std::shared_ptr GCSProcessor::getCredentials(const api::core::ProcessContext& context) { + if (const auto gcp_credentials_controller_service = api::utils::parseOptionalControllerService(context, + GCPCredentials)) { return gcp_credentials_controller_service->getCredentials(); } return nullptr; } -void GCSProcessor::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { - if (auto number_of_retries = utils::parseOptionalU64Property(context, NumberOfRetries)) { +MinifiStatus GCSProcessor::onScheduleImpl(api::core::ProcessContext& context) { + if (const auto number_of_retries = api::utils::parseOptionalU64Property(context, NumberOfRetries)) { retry_policy_ = std::make_shared(gsl::narrow(*number_of_retries)); } gcp_credentials_ = getCredentials(context); if (!gcp_credentials_) { - throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing GCP Credentials"); + logger_->log_error("Couldnt find valid credentials"); + return MINIFI_STATUS_UNKNOWN_ERROR; } - endpoint_url_ = context.getProperty(EndpointOverrideURL) | utils::toOptional(); + endpoint_url_ = context.getProperty(EndpointOverrideURL, nullptr) | utils::toOptional(); if (endpoint_url_) logger_->log_debug("Endpoint overwritten: {}", *endpoint_url_); - auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID()); - if (proxy_controller_service && proxy_controller_service->getProxyType() != minifi::controllers::ProxyType::DIRECT) { + const auto proxy_data = context.getProxyData(ProxyConfigurationService) | utils::orThrow("Couldnt query ProxyConfigurationService"); + if (proxy_data && proxy_data->proxy_type != api::utils::ProxyType::DIRECT) { logger_->log_debug("Proxy configuration is set for GCS processor"); proxy_ = google::cloud::ProxyConfig{}; - proxy_->set_scheme(minifi::utils::string::startsWith(proxy_controller_service->getHost(), "https") ? "https" : "http"); - auto proxy_host = proxy_controller_service->getHost(); + proxy_->set_scheme(minifi::utils::string::startsWith(proxy_data->host, "https") ? "https" : "http"); + auto proxy_host = proxy_data->host; constexpr std::string_view https_prefix = "https://"; constexpr std::string_view http_prefix = "http://"; if (minifi::utils::string::startsWith(proxy_host, https_prefix)) { @@ -64,12 +62,13 @@ void GCSProcessor::onSchedule(core::ProcessContext& context, core::ProcessSessio proxy_host = proxy_host.substr(http_prefix.size()); } proxy_->set_hostname(proxy_host); - proxy_->set_port(std::to_string(proxy_controller_service->getPort())); - if (auto proxy_credentials = proxy_controller_service->getProxyCredentials()) { + proxy_->set_port(std::to_string(proxy_data->port)); + if (auto proxy_credentials = proxy_data->proxy_credentials) { proxy_->set_username(proxy_credentials->username); proxy_->set_password(proxy_credentials->password); } } + return MINIFI_STATUS_SUCCESS; } gcs::Client GCSProcessor::getClient() const { diff --git a/extensions/gcp/processors/GCSProcessor.h b/extensions/gcp/processors/GCSProcessor.h index af6a92aced..a05dde4212 100644 --- a/extensions/gcp/processors/GCSProcessor.h +++ b/extensions/gcp/processors/GCSProcessor.h @@ -16,24 +16,21 @@ */ #pragma once -#include #include -#include #include +#include #include "../controllerservices/GCPCredentialsControllerService.h" -#include "minifi-cpp/core/logging/Logger.h" -#include "core/ProcessorImpl.h" -#include "minifi-cpp/core/PropertyDefinition.h" +#include "api/core/ProcessorImpl.h" #include "core/PropertyDefinitionBuilder.h" -#include "minifi-cpp/core/PropertyValidator.h" #include "google/cloud/credentials.h" #include "google/cloud/storage/client.h" #include "google/cloud/storage/retry_policy.h" -#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" +#include "minifi-cpp/core/PropertyDefinition.h" +#include "minifi-cpp/core/PropertyValidator.h" namespace org::apache::nifi::minifi::extensions::gcp { -class GCSProcessor : public core::ProcessorImpl { +class GCSProcessor : public api::core::ProcessorImpl { public: using ProcessorImpl::ProcessorImpl; @@ -56,7 +53,7 @@ class GCSProcessor : public core::ProcessorImpl { .build(); EXTENSIONAPI static constexpr auto ProxyConfigurationService = core::PropertyDefinitionBuilder<>::createProperty("Proxy Configuration Service") .withDescription("Specifies the Proxy Configuration Controller Service to proxy network requests.") - .withAllowedTypes() + .withAllowedType() .build(); EXTENSIONAPI static constexpr auto Properties = std::to_array({ GCPCredentials, @@ -65,12 +62,11 @@ class GCSProcessor : public core::ProcessorImpl { ProxyConfigurationService }); - - void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override; - protected: + MinifiStatus onScheduleImpl(api::core::ProcessContext& context) override; + virtual google::cloud::storage::Client getClient() const; - std::shared_ptr getCredentials(core::ProcessContext& context) const; + static std::shared_ptr getCredentials(const api::core::ProcessContext& context); std::optional endpoint_url_; std::shared_ptr gcp_credentials_; diff --git a/extensions/gcp/processors/ListGCSBucket.cpp b/extensions/gcp/processors/ListGCSBucket.cpp index 13426e4419..ac4cadd40e 100644 --- a/extensions/gcp/processors/ListGCSBucket.cpp +++ b/extensions/gcp/processors/ListGCSBucket.cpp @@ -17,47 +17,44 @@ #include "ListGCSBucket.h" -#include "utils/ProcessorConfigUtils.h" - #include "../GCPAttributes.h" -#include "minifi-cpp/core/FlowFile.h" -#include "minifi-cpp/core/ProcessContext.h" -#include "core/ProcessSession.h" -#include "core/Resource.h" +#include "api/core/ProcessContext.h" +#include "api/core/ProcessSession.h" +#include "api/core/Resource.h" +#include "api/utils/ProcessorConfigUtils.h" +#include "minifi-cpp/core/SpecialFlowAttribute.h" namespace gcs = ::google::cloud::storage; namespace org::apache::nifi::minifi::extensions::gcp { -void ListGCSBucket::initialize() { - setSupportedProperties(Properties); - setSupportedRelationships(Relationships); -} - -void ListGCSBucket::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { - GCSProcessor::onSchedule(context, session_factory); - bucket_ = utils::parseProperty(context, Bucket); +MinifiStatus ListGCSBucket::onScheduleImpl(api::core::ProcessContext& context) { + const auto status = GCSProcessor::onScheduleImpl(context); + if (status != MinifiStatus::MINIFI_STATUS_SUCCESS) { + return status; + } + bucket_ = api::utils::parseProperty(context, Bucket); + return MinifiStatus::MINIFI_STATUS_SUCCESS; } -void ListGCSBucket::onTrigger(core::ProcessContext& context, core::ProcessSession& session) { +MinifiStatus ListGCSBucket::onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) { gsl_Expects(gcp_credentials_); gcs::Client client = getClient(); - auto list_all_versions = utils::parseOptionalBoolProperty(context, ListAllVersions); + auto list_all_versions = api::utils::parseOptionalBoolProperty(context, ListAllVersions); gcs::Versions versions = (list_all_versions && *list_all_versions) ? gcs::Versions(true) : gcs::Versions(false); auto objects_in_bucket = client.ListObjects(bucket_, versions); for (const auto& object_in_bucket : objects_in_bucket) { if (object_in_bucket.ok()) { auto flow_file = session.create(); - flow_file->updateAttribute(core::SpecialFlowAttribute::FILENAME, object_in_bucket->name()); - setAttributesFromObjectMetadata(*flow_file, *object_in_bucket); - session.transfer(flow_file, Success); + session.setAttribute(flow_file, core::SpecialFlowAttribute::FILENAME, object_in_bucket->name()); + setAttributesFromObjectMetadata(flow_file, *object_in_bucket, session); + session.transfer(std::move(flow_file), Success); } else { logger_->log_error("Invalid object in bucket {}", bucket_); } } + return MinifiStatus::MINIFI_STATUS_SUCCESS; } -REGISTER_RESOURCE(ListGCSBucket, Processor); - } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/ListGCSBucket.h b/extensions/gcp/processors/ListGCSBucket.h index 36c4ac1e01..e654a314d7 100644 --- a/extensions/gcp/processors/ListGCSBucket.h +++ b/extensions/gcp/processors/ListGCSBucket.h @@ -17,19 +17,17 @@ #pragma once -#include #include -#include #include "../GCPAttributes.h" #include "GCSProcessor.h" -#include "core/logging/LoggerFactory.h" #include "minifi-cpp/core/OutputAttributeDefinition.h" #include "minifi-cpp/core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "minifi-cpp/core/PropertyValidator.h" #include "minifi-cpp/core/RelationshipDefinition.h" #include "utils/ArrayUtils.h" +#include "minifi-cpp/core/Annotation.h" namespace org::apache::nifi::minifi::extensions::gcp { @@ -44,7 +42,6 @@ inline constexpr auto FILENAME_OUTPUT_ATTRIBUTE_DESCRIPTION = utils::array_to_st class ListGCSBucket : public GCSProcessor { public: using GCSProcessor::GCSProcessor; - ~ListGCSBucket() override = default; EXTENSIONAPI static constexpr const char* Description = "Retrieves a listing of objects from an GCS bucket. " "For each object that is listed, creates a FlowFile that represents the object so that it can be fetched in conjunction with FetchGCSObject."; @@ -120,11 +117,9 @@ class ListGCSBucket : public GCSProcessor { EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_FORBIDDEN; EXTENSIONAPI static constexpr bool IsSingleThreaded = true; - ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_PROCESSORS - - void initialize() override; - void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override; - void onTrigger(core::ProcessContext& context, core::ProcessSession& session) override; + protected: + MinifiStatus onScheduleImpl(api::core::ProcessContext& context) override; + MinifiStatus onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) override; private: std::string bucket_; diff --git a/extensions/gcp/processors/PutGCSObject.cpp b/extensions/gcp/processors/PutGCSObject.cpp index a05380f32d..97178eb9ac 100644 --- a/extensions/gcp/processors/PutGCSObject.cpp +++ b/extensions/gcp/processors/PutGCSObject.cpp @@ -19,12 +19,12 @@ #include -#include "core/Resource.h" -#include "minifi-cpp/core/FlowFile.h" -#include "minifi-cpp/core/ProcessContext.h" -#include "core/ProcessSession.h" #include "../GCPAttributes.h" -#include "utils/ProcessorConfigUtils.h" +#include "api/core/ProcessContext.h" +#include "api/core/ProcessSession.h" +#include "api/core/Resource.h" +#include "api/utils/ProcessorConfigUtils.h" +#include "minifi-cpp/io/InputStream.h" namespace gcs = ::google::cloud::storage; @@ -102,80 +102,79 @@ class UploadToGCSCallback { } // namespace -void PutGCSObject::initialize() { - setSupportedProperties(Properties); - setSupportedRelationships(Relationships); -} - +MinifiStatus PutGCSObject::onScheduleImpl(api::core::ProcessContext& context) { + const auto status = GCSProcessor::onScheduleImpl(context); + if (status != MinifiStatus::MINIFI_STATUS_SUCCESS) { + return status; + } -void PutGCSObject::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { - GCSProcessor::onSchedule(context, session_factory); - if (auto encryption_key = context.getProperty(EncryptionKey)) { + if (auto encryption_key = context.getProperty(EncryptionKey, nullptr)) { try { encryption_key_ = gcs::EncryptionKey::FromBase64Key(*encryption_key); } catch (const google::cloud::RuntimeStatusError&) { - throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Could not decode the base64-encoded encryption key from property " + std::string(EncryptionKey.name)); + logger_->log_error("Could not decode the base64-encoded encryption key from property {}", std::string(EncryptionKey.name)); + return MINIFI_STATUS_UNKNOWN_ERROR; } } + return MINIFI_STATUS_SUCCESS; } -void PutGCSObject::onTrigger(core::ProcessContext& context, core::ProcessSession& session) { +MinifiStatus PutGCSObject::onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) { gsl_Expects(gcp_credentials_); auto flow_file = session.get(); if (!flow_file) { - context.yield(); - return; + return MINIFI_STATUS_PROCESSOR_YIELD; } - auto bucket = context.getProperty(Bucket, flow_file.get()); + auto bucket = api::utils::parseOptionalProperty(context, Bucket, &flow_file); + if (!bucket || bucket->empty()) { logger_->log_error("Missing bucket name"); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } - auto object_name = context.getProperty(Key, flow_file.get()); + auto object_name = api::utils::parseOptionalProperty(context, Key, &flow_file); if (!object_name || object_name->empty()) { logger_->log_error("Missing object name"); - session.transfer(flow_file, Failure); - return; + session.transfer(std::move(flow_file), Failure); + return MINIFI_STATUS_SUCCESS; } gcs::Client client = getClient(); UploadToGCSCallback callback(client, *bucket, *object_name); - if (auto crc32_checksum = context.getProperty(Crc32cChecksum, flow_file.get())) { + if (auto crc32_checksum = api::utils::parseOptionalProperty(context, Crc32cChecksum, &flow_file)) { callback.setCrc32CChecksumValue(*crc32_checksum); } - if (auto md5_hash = context.getProperty(MD5Hash, flow_file.get())) { + if (auto md5_hash = api::utils::parseOptionalProperty(context, MD5Hash, &flow_file)) { callback.setHashValue(*md5_hash); } - auto content_type = context.getProperty(ContentType, flow_file.get()); + auto content_type = api::utils::parseOptionalProperty(context, ContentType, &flow_file); if (content_type && !content_type->empty()) callback.setContentType(*content_type); - if (auto predefined_acl = utils::parseOptionalEnumProperty(context, ObjectACL)) + if (auto predefined_acl = api::utils::parseOptionalEnumProperty(context, ObjectACL)) callback.setPredefinedAcl(*predefined_acl); - callback.setIfGenerationMatch(utils::parseOptionalBoolProperty(context, OverwriteObject)); + callback.setIfGenerationMatch(api::utils::parseOptionalBoolProperty(context, OverwriteObject)); callback.setEncryptionKey(encryption_key_); session.read(flow_file, std::ref(callback)); auto& result = callback.getResult(); if (!result.ok()) { - flow_file->setAttribute(GCS_STATUS_MESSAGE, result.status().message()); - flow_file->setAttribute(GCS_ERROR_REASON, result.status().error_info().reason()); - flow_file->setAttribute(GCS_ERROR_DOMAIN, result.status().error_info().domain()); + session.setAttribute(flow_file, GCS_STATUS_MESSAGE, result.status().message()); + session.setAttribute(flow_file, GCS_ERROR_REASON, result.status().error_info().reason()); + session.setAttribute(flow_file, GCS_ERROR_DOMAIN, result.status().error_info().domain()); logger_->log_error("Failed to upload to Google Cloud Storage {} {}", result.status().message(), result.status().error_info().reason()); - session.transfer(flow_file, Failure); + session.transfer(std::move(flow_file), Failure); } else { - setAttributesFromObjectMetadata(*flow_file, *result); - session.transfer(flow_file, Success); + setAttributesFromObjectMetadata(flow_file, *result, session); + session.transfer(std::move(flow_file), Success); } + return MINIFI_STATUS_SUCCESS; } -REGISTER_RESOURCE(PutGCSObject, Processor); - } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/PutGCSObject.h b/extensions/gcp/processors/PutGCSObject.h index 58d7675fa5..73d8c82183 100644 --- a/extensions/gcp/processors/PutGCSObject.h +++ b/extensions/gcp/processors/PutGCSObject.h @@ -19,17 +19,16 @@ #include #include -#include #include "../GCPAttributes.h" #include "GCSProcessor.h" #include "minifi-cpp/core/PropertyDefinition.h" #include "minifi-cpp/core/PropertyValidator.h" #include "minifi-cpp/core/RelationshipDefinition.h" -#include "core/logging/LoggerFactory.h" #include "utils/ArrayUtils.h" #include "utils/Enum.h" #include "google/cloud/storage/well_known_headers.h" +#include "minifi-cpp/core/Annotation.h" namespace org::apache::nifi::minifi::extensions::gcp::put_gcs_object { enum class PredefinedAcl { @@ -73,7 +72,6 @@ namespace org::apache::nifi::minifi::extensions::gcp { class PutGCSObject : public GCSProcessor { public: using GCSProcessor::GCSProcessor; - ~PutGCSObject() override = default; EXTENSIONAPI static constexpr const char* Description = "Puts flow files to a Google Cloud Storage Bucket."; @@ -191,11 +189,9 @@ class PutGCSObject : public GCSProcessor { EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; EXTENSIONAPI static constexpr bool IsSingleThreaded = false; - ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_PROCESSORS - - void initialize() override; - void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override; - void onTrigger(core::ProcessContext& context, core::ProcessSession& session) override; + protected: + MinifiStatus onScheduleImpl(api::core::ProcessContext& context) override; + MinifiStatus onTriggerImpl(api::core::ProcessContext& context, api::core::ProcessSession& session) override; private: google::cloud::storage::EncryptionKey encryption_key_; diff --git a/extensions/gcp/tests/CMakeLists.txt b/extensions/gcp/tests/CMakeLists.txt index 0acc5241bf..89fe308e95 100644 --- a/extensions/gcp/tests/CMakeLists.txt +++ b/extensions/gcp/tests/CMakeLists.txt @@ -36,8 +36,8 @@ FOREACH(testfile ${GCS_TESTS}) createTests("${testfilename}") target_link_libraries(${testfilename} minifi-gcp) - target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} gtest_main gmock) + target_link_libraries(${testfilename} libminifi-c-unittest) gtest_add_tests(TARGET "${testfilename}") ENDFOREACH() diff --git a/extensions/gcp/tests/DeleteGCSObjectTests.cpp b/extensions/gcp/tests/DeleteGCSObjectTests.cpp index bde7e010ff..aee457a60c 100644 --- a/extensions/gcp/tests/DeleteGCSObjectTests.cpp +++ b/extensions/gcp/tests/DeleteGCSObjectTests.cpp @@ -14,60 +14,65 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "../processors/DeleteGCSObject.h" #include "../controllerservices/GCPCredentialsControllerService.h" +#include "../processors/DeleteGCSObject.h" +#include "CProcessorTestUtils.h" #include "GCPAttributes.h" #include "core/Resource.h" -#include "unit/SingleProcessorTestController.h" -#include "google/cloud/storage/testing/mock_client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/testing/canonical_errors.h" +#include "google/cloud/storage/testing/mock_client.h" #include "unit/ProcessorUtils.h" +#include "unit/SingleProcessorTestController.h" namespace gcs = ::google::cloud::storage; -namespace minifi_gcp = org::apache::nifi::minifi::extensions::gcp; +namespace minifi_gcp = minifi::extensions::gcp; -using DeleteGCSObject = org::apache::nifi::minifi::extensions::gcp::DeleteGCSObject; -using GCPCredentialsControllerService = org::apache::nifi::minifi::extensions::gcp::GCPCredentialsControllerService; +using DeleteGCSObject = minifi::extensions::gcp::DeleteGCSObject; +using GCPCredentialsControllerService = minifi::extensions::gcp::GCPCredentialsControllerService; using DeleteObjectRequest = gcs::internal::DeleteObjectRequest; -using ::google::cloud::storage::testing::canonical_errors::TransientError; using ::google::cloud::storage::testing::canonical_errors::PermanentError; +using ::google::cloud::storage::testing::canonical_errors::TransientError; namespace { class DeleteGCSObjectMocked : public DeleteGCSObject { - using org::apache::nifi::minifi::extensions::gcp::DeleteGCSObject::DeleteGCSObject; + using DeleteGCSObject::DeleteGCSObject; + public: - static constexpr const char* Description = "DeleteGCSObjectMocked"; + DeleteGCSObjectMocked(minifi::core::ProcessorMetadata metadata, std::shared_ptr mock_client) + : DeleteGCSObject(std::move(metadata)), + mock_client_(std::move(mock_client)) {} - gcs::Client getClient() const override { - return gcs::testing::UndecoratedClientFromMock(mock_client_); - } - std::shared_ptr mock_client_ = std::make_shared(); + protected: + gcs::Client getClient() const override { return gcs::testing::UndecoratedClientFromMock(mock_client_); } + std::shared_ptr mock_client_; }; -REGISTER_RESOURCE(DeleteGCSObjectMocked, Processor); } // namespace class DeleteGCSObjectTests : public ::testing::Test { - public: + protected: void SetUp() override { - delete_gcs_object_ = test_controller_.getProcessor(); - gcp_credentials_node_ = test_controller_.plan->addController("GCPCredentialsControllerService", "gcp_credentials_controller_service"); - test_controller_.plan->setProperty(gcp_credentials_node_, - GCPCredentialsControllerService::CredentialsLoc, - magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS)); - test_controller_.plan->setProperty(delete_gcs_object_, - DeleteGCSObject::GCPCredentials, - "gcp_credentials_controller_service"); + const auto gcp_credential_controller_service = + minifi::test::utils::make_custom_c_controller_service(core::ControllerServiceMetadata{utils::Identifier{}, + "GCPCredentialsControllerService", + logging::LoggerFactory::getLogger()}); + gcp_credentials_node_ = test_controller_.plan->addController("gcp_credentials_controller_service", gcp_credential_controller_service); + EXPECT_TRUE(gcp_credential_controller_service->setProperty(GCPCredentialsControllerService::CredentialsLoc.name, + std::string(magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS)))); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(DeleteGCSObject::GCPCredentials.name, "gcp_credentials_controller_service")); } - org::apache::nifi::minifi::test::SingleProcessorTestController test_controller_{minifi::test::utils::make_processor("DeleteGCSObjectMocked")}; - std::shared_ptr gcp_credentials_node_; - TypedProcessorWrapper delete_gcs_object_ = nullptr; + public: + std::shared_ptr mock_client_ = std::make_shared(); + minifi::test::SingleProcessorTestController test_controller_{minifi::test::utils::make_custom_c_processor( + core::ProcessorMetadata{utils::Identifier{}, "DeleteGCSObjectMocked", logging::LoggerFactory::getLogger()}, + mock_client_)}; + std::shared_ptr gcp_credentials_node_; }; TEST_F(DeleteGCSObjectTests, MissingBucket) { - EXPECT_CALL(*delete_gcs_object_.get().mock_client_, CreateResumableUpload).Times(0); - EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::Bucket, "")); + EXPECT_CALL(*mock_client_, CreateResumableUpload).Times(0); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(DeleteGCSObject::Bucket.name, "")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(0, result.at(DeleteGCSObject::Success).size()); ASSERT_EQ(1, result.at(DeleteGCSObject::Failure).size()); @@ -77,9 +82,9 @@ TEST_F(DeleteGCSObjectTests, MissingBucket) { } TEST_F(DeleteGCSObjectTests, ServerGivesPermaError) { - EXPECT_CALL(*delete_gcs_object_.get().mock_client_, DeleteObject) + EXPECT_CALL(*mock_client_, DeleteObject) .WillOnce(testing::Return(PermanentError())); - EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::Bucket, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(DeleteGCSObject::Bucket.name, "bucket-from-property")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(0, result.at(DeleteGCSObject::Success).size()); ASSERT_EQ(1, result.at(DeleteGCSObject::Failure).size()); @@ -89,9 +94,9 @@ TEST_F(DeleteGCSObjectTests, ServerGivesPermaError) { } TEST_F(DeleteGCSObjectTests, ServerGivesTransientErrors) { - EXPECT_CALL(*delete_gcs_object_.get().mock_client_, DeleteObject).WillOnce(testing::Return(TransientError())); - EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::NumberOfRetries, "1")); - EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::Bucket, "bucket-from-property")); + EXPECT_CALL(*mock_client_, DeleteObject).WillOnce(testing::Return(TransientError())); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(DeleteGCSObject::NumberOfRetries.name, "1")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(DeleteGCSObject::Bucket.name, "bucket-from-property")); const auto& result = test_controller_.trigger("hello world", {{std::string(minifi_gcp::GCS_BUCKET_ATTR), "bucket-from-attribute"}}); EXPECT_EQ(0, result.at(DeleteGCSObject::Success).size()); ASSERT_EQ(1, result.at(DeleteGCSObject::Failure).size()); @@ -100,9 +105,8 @@ TEST_F(DeleteGCSObjectTests, ServerGivesTransientErrors) { EXPECT_EQ("hello world", test_controller_.plan->getContent(result.at(DeleteGCSObject::Failure)[0])); } - TEST_F(DeleteGCSObjectTests, HandlingSuccessfullDeletion) { - EXPECT_CALL(*delete_gcs_object_.get().mock_client_, DeleteObject) + EXPECT_CALL(*mock_client_, DeleteObject) .WillOnce([](DeleteObjectRequest const& request) { EXPECT_EQ("bucket-from-attribute", request.bucket_name()); EXPECT_TRUE(request.HasOption()); @@ -110,7 +114,7 @@ TEST_F(DeleteGCSObjectTests, HandlingSuccessfullDeletion) { EXPECT_EQ(23, request.GetOption().value()); return google::cloud::make_status_or(gcs::internal::EmptyResponse{}); }); - EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::ObjectGeneration, "${gcs.generation}")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(DeleteGCSObject::ObjectGeneration.name, "${gcs.generation}")); const auto& result = test_controller_.trigger("hello world", {{std::string(minifi_gcp::GCS_BUCKET_ATTR), "bucket-from-attribute"}, {std::string(minifi_gcp::GCS_GENERATION), "23"}}); ASSERT_EQ(1, result.at(DeleteGCSObject::Success).size()); EXPECT_EQ(0, result.at(DeleteGCSObject::Failure).size()); @@ -118,13 +122,13 @@ TEST_F(DeleteGCSObjectTests, HandlingSuccessfullDeletion) { } TEST_F(DeleteGCSObjectTests, EmptyGeneration) { - EXPECT_CALL(*delete_gcs_object_.get().mock_client_, DeleteObject) + EXPECT_CALL(*mock_client_, DeleteObject) .WillOnce([](DeleteObjectRequest const& request) { EXPECT_EQ("bucket-from-attribute", request.bucket_name()); EXPECT_FALSE(request.HasOption()); return google::cloud::make_status_or(gcs::internal::EmptyResponse{}); }); - EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::ObjectGeneration, "${gcs.generation}")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(DeleteGCSObject::ObjectGeneration.name, "${gcs.generation}")); const auto& result = test_controller_.trigger("hello world", {{std::string(minifi_gcp::GCS_BUCKET_ATTR), "bucket-from-attribute"}}); ASSERT_EQ(1, result.at(DeleteGCSObject::Success).size()); EXPECT_EQ(0, result.at(DeleteGCSObject::Failure).size()); @@ -132,7 +136,7 @@ TEST_F(DeleteGCSObjectTests, EmptyGeneration) { } TEST_F(DeleteGCSObjectTests, InvalidGeneration) { - EXPECT_TRUE(test_controller_.plan->setProperty(delete_gcs_object_, DeleteGCSObject::ObjectGeneration, "${gcs.generation}")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(DeleteGCSObject::ObjectGeneration.name, "${gcs.generation}")); const auto& result = test_controller_.trigger("hello world", {{std::string(minifi_gcp::GCS_BUCKET_ATTR), "bucket-from-attribute"}, {std::string(minifi_gcp::GCS_GENERATION), "23 banana"}}); ASSERT_EQ(0, result.at(DeleteGCSObject::Success).size()); EXPECT_EQ(1, result.at(DeleteGCSObject::Failure).size()); diff --git a/extensions/gcp/tests/FetchGCSObjectTests.cpp b/extensions/gcp/tests/FetchGCSObjectTests.cpp index ea87c7e63d..764d57ffeb 100644 --- a/extensions/gcp/tests/FetchGCSObjectTests.cpp +++ b/extensions/gcp/tests/FetchGCSObjectTests.cpp @@ -14,56 +14,63 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "../processors/FetchGCSObject.h" +#include + #include "../controllerservices/GCPCredentialsControllerService.h" +#include "../processors/FetchGCSObject.h" +#include "CProcessorTestUtils.h" #include "GCPAttributes.h" #include "core/Resource.h" -#include "unit/SingleProcessorTestController.h" -#include "google/cloud/storage/testing/mock_client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/testing/canonical_errors.h" +#include "google/cloud/storage/testing/mock_client.h" #include "unit/ProcessorUtils.h" +#include "unit/SingleProcessorTestController.h" namespace gcs = ::google::cloud::storage; -namespace minifi_gcp = org::apache::nifi::minifi::extensions::gcp; +namespace minifi_gcp = minifi::extensions::gcp; -using FetchGCSObject = org::apache::nifi::minifi::extensions::gcp::FetchGCSObject; -using GCPCredentialsControllerService = org::apache::nifi::minifi::extensions::gcp::GCPCredentialsControllerService; +using FetchGCSObject = minifi::extensions::gcp::FetchGCSObject; +using GCPCredentialsControllerService = minifi::extensions::gcp::GCPCredentialsControllerService; namespace { class FetchGCSObjectMocked : public FetchGCSObject { - using org::apache::nifi::minifi::extensions::gcp::FetchGCSObject::FetchGCSObject; + using FetchGCSObject::FetchGCSObject; + public: - static constexpr const char* Description = "FetchGCSObjectMocked"; + FetchGCSObjectMocked(minifi::core::ProcessorMetadata metadata, std::shared_ptr mock_client) + : FetchGCSObject(std::move(metadata)), + mock_client_(std::move(mock_client)) {} - gcs::Client getClient() const override { - return gcs::testing::UndecoratedClientFromMock(mock_client_); - } - std::shared_ptr mock_client_ = std::make_shared(); + protected: + gcs::Client getClient() const override { return gcs::testing::UndecoratedClientFromMock(mock_client_); } + std::shared_ptr mock_client_; }; -REGISTER_RESOURCE(FetchGCSObjectMocked, Processor); } // namespace class FetchGCSObjectTests : public ::testing::Test { - public: + protected: void SetUp() override { - fetch_gcs_object_ = test_controller_.getProcessor(); - gcp_credentials_node_ = test_controller_.plan->addController("GCPCredentialsControllerService", "gcp_credentials_controller_service"); - test_controller_.plan->setProperty(gcp_credentials_node_, - GCPCredentialsControllerService::CredentialsLoc, - magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS)); - test_controller_.plan->setProperty(fetch_gcs_object_, - FetchGCSObject::GCPCredentials, - "gcp_credentials_controller_service"); + const auto gcp_credential_controller_service = + minifi::test::utils::make_custom_c_controller_service(core::ControllerServiceMetadata{utils::Identifier{}, + "GCPCredentialsControllerService", + logging::LoggerFactory::getLogger()}); + gcp_credentials_node_ = test_controller_.plan->addController("gcp_credentials_controller_service", gcp_credential_controller_service); + EXPECT_TRUE(gcp_credential_controller_service->setProperty(GCPCredentialsControllerService::CredentialsLoc.name, + std::string(magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS)))); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(FetchGCSObjectMocked::GCPCredentials.name, "gcp_credentials_controller_service")); } - TypedProcessorWrapper fetch_gcs_object_; - org::apache::nifi::minifi::test::SingleProcessorTestController test_controller_{minifi::test::utils::make_processor("FetchGCSObjectMocked")}; - std::shared_ptr gcp_credentials_node_; + + public: + std::shared_ptr mock_client_ = std::make_shared(); + minifi::test::SingleProcessorTestController test_controller_{minifi::test::utils::make_custom_c_processor( + core::ProcessorMetadata{utils::Identifier{}, "FetchGCSObjectMocked", logging::LoggerFactory::getLogger()}, mock_client_)}; + std::shared_ptr gcp_credentials_node_; }; TEST_F(FetchGCSObjectTests, MissingBucket) { - EXPECT_CALL(*fetch_gcs_object_.get().mock_client_, CreateResumableUpload).Times(0); - EXPECT_TRUE(test_controller_.plan->setProperty(fetch_gcs_object_, FetchGCSObject::Bucket, "")); + EXPECT_CALL(*mock_client_, CreateResumableUpload).Times(0); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(FetchGCSObject::Bucket.name, "")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(0, result.at(FetchGCSObject::Success).size()); ASSERT_EQ(1, result.at(FetchGCSObject::Failure).size()); @@ -73,7 +80,7 @@ TEST_F(FetchGCSObjectTests, MissingBucket) { } TEST_F(FetchGCSObjectTests, ServerError) { - EXPECT_CALL(*fetch_gcs_object_.get().mock_client_, ReadObject) + EXPECT_CALL(*mock_client_, ReadObject) .WillOnce([](gcs::internal::ReadObjectRangeRequest const& request) { EXPECT_EQ(request.bucket_name(), "bucket-from-property") << request; auto mock_source = std::make_unique(); @@ -88,7 +95,7 @@ TEST_F(FetchGCSObjectTests, ServerError) { std::unique_ptr object_read_source = std::move(mock_source); return google::cloud::make_status_or(std::move(object_read_source)); }); - EXPECT_TRUE(test_controller_.plan->setProperty(fetch_gcs_object_, FetchGCSObject::Bucket, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(FetchGCSObject::Bucket.name, "bucket-from-property")); const auto& result = test_controller_.trigger("hello world", {{std::string(minifi_gcp::GCS_BUCKET_ATTR), "bucket-from-attribute"}}); EXPECT_EQ(0, result.at(FetchGCSObject::Success).size()); ASSERT_EQ(1, result.at(FetchGCSObject::Failure).size()); @@ -97,7 +104,7 @@ TEST_F(FetchGCSObjectTests, ServerError) { } TEST_F(FetchGCSObjectTests, HappyPath) { - std::string const text = "stored text"; + constexpr std::string_view text = "stored text"; std::size_t offset = 0; // Simulate a Read() call in the MockObjectReadSource object created below auto simulate_read = [&text, &offset](void* buf, std::size_t n) { @@ -107,13 +114,13 @@ TEST_F(FetchGCSObjectTests, HappyPath) { return gcs::internal::ReadSourceResult{ l, gcs::internal::HttpResponse{200, {}, {}}}; }; - EXPECT_CALL(*fetch_gcs_object_.get().mock_client_, ReadObject) + EXPECT_CALL(*mock_client_, ReadObject) .WillOnce([&](gcs::internal::ReadObjectRangeRequest const& request) { EXPECT_EQ(request.bucket_name(), "bucket-from-attribute") << request; EXPECT_TRUE(request.HasOption()); EXPECT_TRUE(request.GetOption().has_value()); EXPECT_EQ(23, request.GetOption().value()); - std::unique_ptr mock_source(new gcs::testing::MockObjectReadSource); + auto mock_source = std::make_unique(); ::testing::InSequence seq; EXPECT_CALL(*mock_source, IsOpen()).WillRepeatedly(testing::Return(true)); EXPECT_CALL(*mock_source, Read).WillOnce(simulate_read); @@ -123,7 +130,7 @@ TEST_F(FetchGCSObjectTests, HappyPath) { std::unique_ptr( std::move(mock_source))); }); - EXPECT_TRUE(test_controller_.plan->setProperty(fetch_gcs_object_, FetchGCSObject::ObjectGeneration, "${gcs.generation}")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(FetchGCSObject::ObjectGeneration.name, "${gcs.generation}")); const auto& result = test_controller_.trigger("hello world", {{std::string(minifi_gcp::GCS_BUCKET_ATTR), "bucket-from-attribute"}, {std::string(minifi_gcp::GCS_GENERATION), "23"}}); ASSERT_EQ(1, result.at(FetchGCSObject::Success).size()); EXPECT_EQ(0, result.at(FetchGCSObject::Failure).size()); @@ -131,7 +138,7 @@ TEST_F(FetchGCSObjectTests, HappyPath) { } TEST_F(FetchGCSObjectTests, EmptyGeneration) { - std::string const text = "stored text"; + constexpr std::string_view text = "stored text"; std::size_t offset = 0; // Simulate a Read() call in the MockObjectReadSource object created below auto simulate_read = [&text, &offset](void* buf, std::size_t n) { @@ -141,11 +148,11 @@ TEST_F(FetchGCSObjectTests, EmptyGeneration) { return gcs::internal::ReadSourceResult{ l, gcs::internal::HttpResponse{200, {}, {}}}; }; - EXPECT_CALL(*fetch_gcs_object_.get().mock_client_, ReadObject) + EXPECT_CALL(*mock_client_, ReadObject) .WillOnce([&](gcs::internal::ReadObjectRangeRequest const& request) { EXPECT_EQ(request.bucket_name(), "bucket-from-attribute") << request; EXPECT_FALSE(request.HasOption()); - std::unique_ptr mock_source(new gcs::testing::MockObjectReadSource); + auto mock_source = std::make_unique(); ::testing::InSequence seq; EXPECT_CALL(*mock_source, IsOpen()).WillRepeatedly(testing::Return(true)); EXPECT_CALL(*mock_source, Read).WillOnce(simulate_read); @@ -155,7 +162,7 @@ TEST_F(FetchGCSObjectTests, EmptyGeneration) { std::unique_ptr( std::move(mock_source))); }); - EXPECT_TRUE(test_controller_.plan->setProperty(fetch_gcs_object_, FetchGCSObject::ObjectGeneration, "${gcs.generation}")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(FetchGCSObject::ObjectGeneration.name, "${gcs.generation}")); const auto& result = test_controller_.trigger("hello world", {{std::string(minifi_gcp::GCS_BUCKET_ATTR), "bucket-from-attribute"}}); ASSERT_EQ(1, result.at(FetchGCSObject::Success).size()); EXPECT_EQ(0, result.at(FetchGCSObject::Failure).size()); @@ -163,7 +170,7 @@ TEST_F(FetchGCSObjectTests, EmptyGeneration) { } TEST_F(FetchGCSObjectTests, InvalidGeneration) { - EXPECT_TRUE(test_controller_.plan->setProperty(fetch_gcs_object_, FetchGCSObject::ObjectGeneration, "${gcs.generation}")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(FetchGCSObject::ObjectGeneration.name, "${gcs.generation}")); const auto& result = test_controller_.trigger("hello world", {{std::string(minifi_gcp::GCS_BUCKET_ATTR), "bucket-from-attribute"}, {std::string(minifi_gcp::GCS_GENERATION), "23 banana"}}); ASSERT_EQ(0, result.at(FetchGCSObject::Success).size()); EXPECT_EQ(1, result.at(FetchGCSObject::Failure).size()); diff --git a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp index 0334cb4d95..bedbac924c 100644 --- a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp +++ b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp @@ -16,19 +16,20 @@ */ #define EXTENSION_LIST "minifi-gcp" // NOLINT(cppcoreguidelines-macro-usage) -#include "unit/TestBase.h" -#include "gtest/gtest.h" #include "../controllerservices/GCPCredentialsControllerService.h" -#include "core/Resource.h" +#include "CProcessorTestUtils.h" #include "core/Processor.h" +#include "core/Resource.h" #include "core/controller/ControllerServiceNode.h" +#include "gtest/gtest.h" #include "rapidjson/document.h" #include "rapidjson/stream.h" #include "rapidjson/writer.h" #include "unit/DummyProcessor.h" +#include "unit/TestBase.h" #include "utils/Environment.h" -namespace minifi_gcp = org::apache::nifi::minifi::extensions::gcp; +namespace minifi_gcp = minifi::extensions::gcp; using GCPCredentialsControllerService = minifi_gcp::GCPCredentialsControllerService; namespace { @@ -70,14 +71,20 @@ std::optional create_mock_json_file(const std::filesystem class GCPCredentialsTests : public ::testing::Test { protected: void SetUp() override { - ASSERT_TRUE(gcp_credentials_node_); - ASSERT_TRUE(gcp_credentials_); plan_->addProcessor("DummyProcessor", "dummy_processor"); + auto gcp_credential_controller_service = + minifi::test::utils::make_custom_c_controller_service(core::ControllerServiceMetadata{utils::Identifier{}, + "GCPCredentialsControllerService", + logging::LoggerFactory::getLogger()}); + gcp_credentials_node_ = plan_->addController("gcp_credentials_controller_service", gcp_credential_controller_service); + gcp_credentials_ = static_cast( + gcp_credentials_node_->getControllerServiceImplementation()->getImpl()); } TestController test_controller_; std::shared_ptr plan_ = test_controller_.createPlan(); - std::shared_ptr gcp_credentials_node_ = plan_->addController("GCPCredentialsControllerService", "gcp_credentials_controller_service"); - std::shared_ptr gcp_credentials_ = gcp_credentials_node_->getControllerServiceImplementation(); + + std::shared_ptr gcp_credentials_node_; + GCPCredentialsControllerService* gcp_credentials_ = nullptr; }; TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithEnv) { @@ -85,13 +92,17 @@ TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithEnv) { auto path = create_mock_json_file(temp_directory); ASSERT_TRUE(path.has_value()); minifi::utils::Environment::setEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", path->string().c_str()); - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_DEFAULT_CREDENTIALS)); + EXPECT_TRUE(plan_->setProperty(gcp_credentials_node_, + GCPCredentialsControllerService::CredentialsLoc, + magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_DEFAULT_CREDENTIALS))); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_NE(nullptr, gcp_credentials_->getCredentials()); } TEST_F(GCPCredentialsTests, CredentialsFromJsonWithoutProperty) { - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_FILE)); + EXPECT_TRUE(plan_->setProperty(gcp_credentials_node_, + GCPCredentialsControllerService::CredentialsLoc, + magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_FILE))); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_EQ(nullptr, gcp_credentials_->getCredentials()); } @@ -100,8 +111,10 @@ TEST_F(GCPCredentialsTests, CredentialsFromJsonWithProperty) { auto temp_directory = test_controller_.createTempDirectory(); auto path = create_mock_json_file(temp_directory); ASSERT_TRUE(path.has_value()); - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_FILE)); - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::JsonFilePath, path->string()); + EXPECT_TRUE(plan_->setProperty(gcp_credentials_node_, + GCPCredentialsControllerService::CredentialsLoc, + magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_FILE))); + ASSERT_TRUE(plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::JsonFilePath, path->string())); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_NE(nullptr, gcp_credentials_->getCredentials()); } @@ -110,32 +123,42 @@ TEST_F(GCPCredentialsTests, CredentialsFromJsonWithInvalidPath) { auto temp_directory = test_controller_.createTempDirectory(); auto path = create_mock_json_file(temp_directory); ASSERT_TRUE(path.has_value()); - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_FILE)); - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::JsonFilePath, "/invalid/path/to/credentials.json"); + EXPECT_TRUE(plan_->setProperty(gcp_credentials_node_, + GCPCredentialsControllerService::CredentialsLoc, + magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_FILE))); + ASSERT_TRUE(plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::JsonFilePath, "/invalid/path/to/credentials.json")); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_EQ(nullptr, gcp_credentials_->getCredentials()); } TEST_F(GCPCredentialsTests, CredentialsFromComputeEngineVM) { - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_COMPUTE_ENGINE_CREDENTIALS)); + EXPECT_TRUE(plan_->setProperty(gcp_credentials_node_, + GCPCredentialsControllerService::CredentialsLoc, + magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_COMPUTE_ENGINE_CREDENTIALS))); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_NE(nullptr, gcp_credentials_->getCredentials()); } TEST_F(GCPCredentialsTests, AnonymousCredentials) { - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS)); + EXPECT_TRUE(plan_->setProperty(gcp_credentials_node_, + GCPCredentialsControllerService::CredentialsLoc, + magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS))); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_NE(nullptr, gcp_credentials_->getCredentials()); } TEST_F(GCPCredentialsTests, CredentialsFromJsonContentsWithoutProperty) { - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_CONTENTS)); + EXPECT_TRUE(plan_->setProperty(gcp_credentials_node_, + GCPCredentialsControllerService::CredentialsLoc, + magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_CONTENTS))); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_EQ(nullptr, gcp_credentials_->getCredentials()); } TEST_F(GCPCredentialsTests, CredentialsFromJsonContentsWithProperty) { - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_CONTENTS)); + EXPECT_TRUE(plan_->setProperty(gcp_credentials_node_, + GCPCredentialsControllerService::CredentialsLoc, + magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_CONTENTS))); plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::JsonContents, create_mock_service_json()); ASSERT_NO_THROW(test_controller_.runSession(plan_)); EXPECT_NE(nullptr, gcp_credentials_->getCredentials()); diff --git a/extensions/gcp/tests/ListGCSBucketTests.cpp b/extensions/gcp/tests/ListGCSBucketTests.cpp index 16b2c8dadb..d286fc7398 100644 --- a/extensions/gcp/tests/ListGCSBucketTests.cpp +++ b/extensions/gcp/tests/ListGCSBucketTests.cpp @@ -14,37 +14,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "../processors/ListGCSBucket.h" #include "../controllerservices/GCPCredentialsControllerService.h" +#include "../processors/ListGCSBucket.h" +#include "CProcessorTestUtils.h" #include "core/Resource.h" -#include "unit/SingleProcessorTestController.h" -#include "google/cloud/storage/testing/mock_client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/testing/canonical_errors.h" +#include "google/cloud/storage/testing/mock_client.h" #include "unit/ProcessorUtils.h" +#include "unit/SingleProcessorTestController.h" namespace gcs = ::google::cloud::storage; -namespace minifi_gcp = org::apache::nifi::minifi::extensions::gcp; +namespace minifi_gcp = minifi::extensions::gcp; -using ListGCSBucket = org::apache::nifi::minifi::extensions::gcp::ListGCSBucket; +using ListGCSBucket = minifi::extensions::gcp::ListGCSBucket; using ListObjectsRequest = gcs::internal::ListObjectsRequest; using ListObjectsResponse = gcs::internal::ListObjectsResponse; -using GCPCredentialsControllerService = org::apache::nifi::minifi::extensions::gcp::GCPCredentialsControllerService; -using ::google::cloud::storage::testing::canonical_errors::TransientError; +using GCPCredentialsControllerService = minifi::extensions::gcp::GCPCredentialsControllerService; using ::google::cloud::storage::testing::canonical_errors::PermanentError; +using ::google::cloud::storage::testing::canonical_errors::TransientError; namespace { class ListGCSBucketMocked : public ListGCSBucket { - using org::apache::nifi::minifi::extensions::gcp::ListGCSBucket::ListGCSBucket; + using ListGCSBucket::ListGCSBucket; + public: - static constexpr const char* Description = "ListGCSBucketMocked"; + ListGCSBucketMocked(minifi::core::ProcessorMetadata metadata, std::shared_ptr mock_client) + : ListGCSBucket(std::move(metadata)), + mock_client_(std::move(mock_client)) {} - gcs::Client getClient() const override { - return gcs::testing::UndecoratedClientFromMock(mock_client_); - } - std::shared_ptr mock_client_ = std::make_shared(); + protected: + gcs::Client getClient() const override { return gcs::testing::UndecoratedClientFromMock(mock_client_); } + std::shared_ptr mock_client_; }; -REGISTER_RESOURCE(ListGCSBucketMocked, Processor); auto CreateObject(int index, int generation = 1) { std::string id = "object-" + std::to_string(index); @@ -64,34 +66,34 @@ auto CreateObject(int index, int generation = 1) { } // namespace class ListGCSBucketTests : public ::testing::Test { - public: + protected: void SetUp() override { - list_gcs_bucket_ = test_controller_.getProcessor(); - gcp_credentials_node_ = test_controller_.plan->addController("GCPCredentialsControllerService", "gcp_credentials_controller_service"); - test_controller_.plan->setProperty(gcp_credentials_node_, - GCPCredentialsControllerService::CredentialsLoc, - magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS)); - test_controller_.plan->setProperty(list_gcs_bucket_, - ListGCSBucket::GCPCredentials, - "gcp_credentials_controller_service"); + const auto gcp_credential_controller_service = + minifi::test::utils::make_custom_c_controller_service(core::ControllerServiceMetadata{utils::Identifier{}, + "GCPCredentialsControllerService", + logging::LoggerFactory::getLogger()}); + gcp_credentials_node_ = test_controller_.plan->addController("gcp_credentials_controller_service", gcp_credential_controller_service); + EXPECT_TRUE(gcp_credential_controller_service->setProperty(GCPCredentialsControllerService::CredentialsLoc.name, + std::string(magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS)))); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(ListGCSBucketMocked::GCPCredentials.name, "gcp_credentials_controller_service")); } - org::apache::nifi::minifi::test::SingleProcessorTestController test_controller_{minifi::test::utils::make_processor("ListGCSBucketMocked")}; - TypedProcessorWrapper list_gcs_bucket_; - std::shared_ptr gcp_credentials_node_; + + public: + std::shared_ptr mock_client_ = std::make_shared(); + minifi::test::SingleProcessorTestController test_controller_{minifi::test::utils::make_custom_c_processor( + core::ProcessorMetadata{utils::Identifier{}, "ListGCSBucketMocked", logging::LoggerFactory::getLogger()}, mock_client_)}; + std::shared_ptr gcp_credentials_node_; }; TEST_F(ListGCSBucketTests, MissingBucket) { - EXPECT_CALL(*list_gcs_bucket_.get().mock_client_, CreateResumableUpload).Times(0); + EXPECT_CALL(*mock_client_, CreateResumableUpload).Times(0); EXPECT_THROW(test_controller_.trigger(), std::runtime_error); } TEST_F(ListGCSBucketTests, ServerGivesPermaError) { - auto return_permanent_error = [](ListObjectsRequest const&) { - return google::cloud::StatusOr(PermanentError()); - }; - EXPECT_CALL(*list_gcs_bucket_.get().mock_client_, ListObjects) - .WillOnce(return_permanent_error); - EXPECT_TRUE(test_controller_.plan->setProperty(list_gcs_bucket_, ListGCSBucket::Bucket, "bucket-from-property")); + auto return_permanent_error = [](ListObjectsRequest const&) { return google::cloud::StatusOr(PermanentError()); }; + EXPECT_CALL(*mock_client_, ListObjects).WillOnce(return_permanent_error); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(ListGCSBucket::Bucket.name, "bucket-from-property")); const auto& result = test_controller_.trigger(); EXPECT_EQ(0, result.at(ListGCSBucket::Success).size()); } @@ -100,15 +102,15 @@ TEST_F(ListGCSBucketTests, ServerGivesTransientErrors) { auto return_temp_error = [](ListObjectsRequest const&) { return google::cloud::StatusOr(TransientError()); }; - EXPECT_CALL(*list_gcs_bucket_.get().mock_client_, ListObjects).WillOnce(return_temp_error); - EXPECT_TRUE(test_controller_.plan->setProperty(list_gcs_bucket_, ListGCSBucket::NumberOfRetries, "1")); - EXPECT_TRUE(test_controller_.plan->setProperty(list_gcs_bucket_, ListGCSBucket::Bucket, "bucket-from-property")); + EXPECT_CALL(*mock_client_, ListObjects).WillOnce(return_temp_error); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(ListGCSBucket::NumberOfRetries.name, "1")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(ListGCSBucket::Bucket.name, "bucket-from-property")); const auto& result = test_controller_.trigger(); EXPECT_EQ(0, result.at(ListGCSBucket::Success).size()); } TEST_F(ListGCSBucketTests, WithoutVersions) { - EXPECT_CALL(*list_gcs_bucket_.get().mock_client_, ListObjects) + EXPECT_CALL(*mock_client_, ListObjects) .WillOnce([](ListObjectsRequest const& req) -> google::cloud::StatusOr { EXPECT_EQ("bucket-from-property", req.bucket_name()); @@ -121,14 +123,13 @@ TEST_F(ListGCSBucketTests, WithoutVersions) { response.items.emplace_back(CreateObject(1, 3)); return response; }); - EXPECT_TRUE(test_controller_.plan->setProperty(list_gcs_bucket_, ListGCSBucket::Bucket, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(ListGCSBucket::Bucket.name, "bucket-from-property")); const auto& result = test_controller_.trigger(); EXPECT_EQ(3, result.at(ListGCSBucket::Success).size()); } - TEST_F(ListGCSBucketTests, WithVersions) { - EXPECT_CALL(*list_gcs_bucket_.get().mock_client_, ListObjects) + EXPECT_CALL(*mock_client_, ListObjects) .WillOnce([](ListObjectsRequest const& req) -> google::cloud::StatusOr { EXPECT_EQ("bucket-from-property", req.bucket_name()); @@ -141,9 +142,8 @@ TEST_F(ListGCSBucketTests, WithVersions) { response.items.emplace_back(CreateObject(3)); return response; }); - EXPECT_TRUE(test_controller_.plan->setProperty(list_gcs_bucket_, ListGCSBucket::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(list_gcs_bucket_, ListGCSBucket::ListAllVersions, "true")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(ListGCSBucket::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(ListGCSBucket::ListAllVersions.name, "true")); const auto& result = test_controller_.trigger(); EXPECT_EQ(3, result.at(ListGCSBucket::Success).size()); } - diff --git a/extensions/gcp/tests/PutGCSObjectTests.cpp b/extensions/gcp/tests/PutGCSObjectTests.cpp index a8355b50f7..01b58104eb 100644 --- a/extensions/gcp/tests/PutGCSObjectTests.cpp +++ b/extensions/gcp/tests/PutGCSObjectTests.cpp @@ -14,56 +14,59 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "../processors/PutGCSObject.h" #include "../controllerservices/GCPCredentialsControllerService.h" +#include "../processors/PutGCSObject.h" +#include "CProcessorTestUtils.h" #include "GCPAttributes.h" #include "core/Resource.h" -#include "unit/SingleProcessorTestController.h" -#include "google/cloud/storage/testing/mock_client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" -#include "google/cloud/storage/retry_policy.h" #include "google/cloud/storage/testing/canonical_errors.h" +#include "google/cloud/storage/testing/mock_client.h" #include "unit/ProcessorUtils.h" +#include "unit/SingleProcessorTestController.h" namespace gcs = ::google::cloud::storage; -namespace minifi_gcp = org::apache::nifi::minifi::extensions::gcp; +namespace minifi_gcp = minifi::extensions::gcp; -using PutGCSObject = org::apache::nifi::minifi::extensions::gcp::PutGCSObject; -using GCPCredentialsControllerService = org::apache::nifi::minifi::extensions::gcp::GCPCredentialsControllerService; +using PutGCSObject = minifi::extensions::gcp::PutGCSObject; +using GCPCredentialsControllerService = minifi::extensions::gcp::GCPCredentialsControllerService; using ResumableUploadRequest = gcs::internal::ResumableUploadRequest; using QueryResumableUploadResponse = gcs::internal::QueryResumableUploadResponse; -using ::google::cloud::storage::testing::canonical_errors::TransientError; using ::google::cloud::storage::testing::canonical_errors::PermanentError; +using ::google::cloud::storage::testing::canonical_errors::TransientError; namespace { class PutGCSObjectMocked : public PutGCSObject { - using org::apache::nifi::minifi::extensions::gcp::PutGCSObject::PutGCSObject; + using PutGCSObject::PutGCSObject; + public: - static constexpr const char* Description = "PutGCSObjectMocked"; + PutGCSObjectMocked(minifi::core::ProcessorMetadata metadata, std::shared_ptr mock_client) + : PutGCSObject(std::move(metadata)), + mock_client_(std::move(mock_client)) {} - gcs::Client getClient() const override { - return gcs::testing::UndecoratedClientFromMock(mock_client_); - } - std::shared_ptr mock_client_ = std::make_shared(); + protected: + gcs::Client getClient() const override { return gcs::testing::UndecoratedClientFromMock(mock_client_); } + std::shared_ptr mock_client_; }; -REGISTER_RESOURCE(PutGCSObjectMocked, Processor); } // namespace class PutGCSObjectTests : public ::testing::Test { - public: + protected: void SetUp() override { - put_gcs_object_ = test_controller_.getProcessor(); - gcp_credentials_node_ = test_controller_.plan->addController("GCPCredentialsControllerService", "gcp_credentials_controller_service"); - test_controller_.plan->setProperty(gcp_credentials_node_, - GCPCredentialsControllerService::CredentialsLoc, - magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS)); - test_controller_.plan->setProperty(put_gcs_object_, - PutGCSObject::GCPCredentials, - "gcp_credentials_controller_service"); + const auto gcp_credential_controller_service = + minifi::test::utils::make_custom_c_controller_service(core::ControllerServiceMetadata{utils::Identifier{}, + "GCPCredentialsControllerService", + logging::LoggerFactory::getLogger()}); + gcp_credentials_node_ = test_controller_.plan->addController("gcp_credentials_controller_service", gcp_credential_controller_service); + EXPECT_TRUE(gcp_credential_controller_service->setProperty(GCPCredentialsControllerService::CredentialsLoc.name, + std::string(magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_ANONYMOUS_CREDENTIALS)))); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObjectMocked::GCPCredentials.name, "gcp_credentials_controller_service")); } - TypedProcessorWrapper put_gcs_object_; - org::apache::nifi::minifi::test::SingleProcessorTestController test_controller_{minifi::test::utils::make_processor("PutGCSObjectMocked")}; - std::shared_ptr gcp_credentials_node_; + + std::shared_ptr mock_client_ = std::make_shared(); + minifi::test::SingleProcessorTestController test_controller_{minifi::test::utils::make_custom_c_processor( + core::ProcessorMetadata{utils::Identifier{}, "PutGCSObjectMocked", logging::LoggerFactory::getLogger()}, mock_client_)}; + std::shared_ptr gcp_credentials_node_; static auto return_upload_done(const ResumableUploadRequest& request) { using ObjectMetadataParser = gcs::internal::ObjectMetadataParser; @@ -75,13 +78,14 @@ class PutGCSObjectTests : public ::testing::Test { metadata_json["customerEncryption"]["encryptionAlgorithm"] = "AES256"; metadata_json["customerEncryption"]["keySha256"] = "zkeXIcAB56dkHp0z1023TQZ+mzm+fZ5JRVgmAQ3bEVE="; } - return testing::Return(google::cloud::make_status_or(QueryResumableUploadResponse{absl::nullopt, *ObjectMetadataParser::FromJson(metadata_json)})); + return testing::Return(google::cloud::make_status_or(QueryResumableUploadResponse{absl::nullopt, + *ObjectMetadataParser::FromJson(metadata_json)})); } }; TEST_F(PutGCSObjectTests, MissingBucket) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload).Times(0); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "")); + EXPECT_CALL(*mock_client_, CreateResumableUpload).Times(0); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(0, result.at(PutGCSObject::Success).size()); ASSERT_EQ(1, result.at(PutGCSObject::Failure).size()); @@ -91,13 +95,13 @@ TEST_F(PutGCSObjectTests, MissingBucket) { } TEST_F(PutGCSObjectTests, BucketFromAttribute) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload) + EXPECT_CALL(*mock_client_, CreateResumableUpload) .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_EQ("bucket-from-attribute", request.bucket_name()); - EXPECT_CALL(*put_gcs_object_.get().mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + EXPECT_CALL(*mock_client_, UploadChunk).WillOnce(return_upload_done(request)); return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "${gcs.bucket}")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "${gcs.bucket}")); const auto& result = test_controller_.trigger("hello world", {{std::string(minifi_gcp::GCS_BUCKET_ATTR), "bucket-from-attribute"}}); ASSERT_EQ(1, result.at(PutGCSObject::Success).size()); EXPECT_EQ(0, result.at(PutGCSObject::Failure).size()); @@ -105,10 +109,10 @@ TEST_F(PutGCSObjectTests, BucketFromAttribute) { } TEST_F(PutGCSObjectTests, ServerGivesTransientErrors) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload).WillOnce(testing::Return(TransientError())); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::NumberOfRetries, "2")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); + EXPECT_CALL(*mock_client_, CreateResumableUpload).WillOnce(testing::Return(TransientError())); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::NumberOfRetries.name, "2")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(0, result.at(PutGCSObject::Success).size()); ASSERT_EQ(1, result.at(PutGCSObject::Failure).size()); @@ -118,9 +122,9 @@ TEST_F(PutGCSObjectTests, ServerGivesTransientErrors) { } TEST_F(PutGCSObjectTests, ServerGivesPermaError) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload).WillOnce(testing::Return(PermanentError())); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); + EXPECT_CALL(*mock_client_, CreateResumableUpload).WillOnce(testing::Return(PermanentError())); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(0, result.at(PutGCSObject::Success).size()); ASSERT_EQ(1, result.at(PutGCSObject::Failure).size()); @@ -130,51 +134,51 @@ TEST_F(PutGCSObjectTests, ServerGivesPermaError) { } TEST_F(PutGCSObjectTests, NonRequiredPropertiesAreMissing) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload) + EXPECT_CALL(*mock_client_, CreateResumableUpload) .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_FALSE(request.HasOption()); EXPECT_FALSE(request.HasOption()); EXPECT_FALSE(request.HasOption()); EXPECT_FALSE(request.HasOption()); - EXPECT_CALL(*put_gcs_object_.get().mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + EXPECT_CALL(*mock_client_, UploadChunk).WillOnce(return_upload_done(request)); return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); const auto& result = test_controller_.trigger("hello world"); EXPECT_EQ(1, result.at(PutGCSObject::Success).size()); EXPECT_EQ(0, result.at(PutGCSObject::Failure).size()); } TEST_F(PutGCSObjectTests, Crc32cMD5LocationTest) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload) + EXPECT_CALL(*mock_client_, CreateResumableUpload) .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption()); EXPECT_EQ("yZRlqg==", request.GetOption().value()); EXPECT_TRUE(request.HasOption()); EXPECT_EQ("XrY7u+Ae7tCTyyK7j1rNww==", request.GetOption().value()); - EXPECT_CALL(*put_gcs_object_.get().mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + EXPECT_CALL(*mock_client_, UploadChunk).WillOnce(return_upload_done(request)); return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::MD5Hash, "${md5}")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Crc32cChecksum, "${crc32c}")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::MD5Hash.name, "${md5}")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Crc32cChecksum.name, "${crc32c}")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); const auto& result = test_controller_.trigger("hello world", {{"crc32c", "yZRlqg=="}, {"md5", "XrY7u+Ae7tCTyyK7j1rNww=="}}); EXPECT_EQ(1, result.at(PutGCSObject::Success).size()); EXPECT_EQ(0, result.at(PutGCSObject::Failure).size()); } TEST_F(PutGCSObjectTests, DontOverwriteTest) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload) + EXPECT_CALL(*mock_client_, CreateResumableUpload) .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption()); - EXPECT_CALL(*put_gcs_object_.get().mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + EXPECT_CALL(*mock_client_, UploadChunk).WillOnce(return_upload_done(request)); return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::OverwriteObject, "false")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::OverwriteObject.name, "false")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); const auto& result = test_controller_.trigger("hello world", {{"crc32c", "yZRlqg=="}, {"md5", "XrY7u+Ae7tCTyyK7j1rNww=="}}); ASSERT_EQ(1, result.at(PutGCSObject::Success).size()); EXPECT_EQ(0, result.at(PutGCSObject::Failure).size()); @@ -182,15 +186,15 @@ TEST_F(PutGCSObjectTests, DontOverwriteTest) { } TEST_F(PutGCSObjectTests, ValidServerSideEncryptionTest) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload) + EXPECT_CALL(*mock_client_, CreateResumableUpload) .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption()); - EXPECT_CALL(*put_gcs_object_.get().mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + EXPECT_CALL(*mock_client_, UploadChunk).WillOnce(return_upload_done(request)); return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::EncryptionKey, "ZW5jcnlwdGlvbl9rZXk=")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::EncryptionKey.name, "ZW5jcnlwdGlvbl9rZXk=")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); const auto& result = test_controller_.trigger("hello world"); ASSERT_EQ(1, result.at(PutGCSObject::Success).size()); EXPECT_EQ(0, result.at(PutGCSObject::Failure).size()); @@ -200,22 +204,22 @@ TEST_F(PutGCSObjectTests, ValidServerSideEncryptionTest) { } TEST_F(PutGCSObjectTests, InvalidServerSideEncryptionTest) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload).Times(0); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::EncryptionKey, "not_base64_key")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); + EXPECT_CALL(*mock_client_, CreateResumableUpload).Times(0); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::EncryptionKey.name, "not_base64_key")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); EXPECT_THROW(test_controller_.trigger("hello world"), minifi::Exception); } TEST_F(PutGCSObjectTests, NoContentType) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload) + EXPECT_CALL(*mock_client_, CreateResumableUpload) .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_FALSE(request.HasOption()); - EXPECT_CALL(*put_gcs_object_.get().mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + EXPECT_CALL(*mock_client_, UploadChunk).WillOnce(return_upload_done(request)); return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); const auto& result = test_controller_.trigger("hello world"); ASSERT_EQ(1, result.at(PutGCSObject::Success).size()); EXPECT_EQ(0, result.at(PutGCSObject::Failure).size()); @@ -223,15 +227,15 @@ TEST_F(PutGCSObjectTests, NoContentType) { } TEST_F(PutGCSObjectTests, ContentTypeFromAttribute) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload) + EXPECT_CALL(*mock_client_, CreateResumableUpload) .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption()); EXPECT_EQ("text/attribute", request.GetOption().value()); - EXPECT_CALL(*put_gcs_object_.get().mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + EXPECT_CALL(*mock_client_, UploadChunk).WillOnce(return_upload_done(request)); return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); const auto& result = test_controller_.trigger("hello world", {{"mime.type", "text/attribute"}}); ASSERT_EQ(1, result.at(PutGCSObject::Success).size()); EXPECT_EQ(0, result.at(PutGCSObject::Failure).size()); @@ -239,16 +243,17 @@ TEST_F(PutGCSObjectTests, ContentTypeFromAttribute) { } TEST_F(PutGCSObjectTests, ObjectACLTest) { - EXPECT_CALL(*put_gcs_object_.get().mock_client_, CreateResumableUpload) + EXPECT_CALL(*mock_client_, CreateResumableUpload) .WillOnce([this](const ResumableUploadRequest& request) { EXPECT_TRUE(request.HasOption()); EXPECT_EQ(gcs::PredefinedAcl::AuthenticatedRead().value(), request.GetOption().value()); - EXPECT_CALL(*put_gcs_object_.get().mock_client_, UploadChunk).WillOnce(return_upload_done(request)); + EXPECT_CALL(*mock_client_, UploadChunk).WillOnce(return_upload_done(request)); return gcs::internal::CreateResumableUploadResponse{"test-only-upload-id"}; }); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Bucket, "bucket-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::Key, "object-name-from-property")); - EXPECT_TRUE(test_controller_.plan->setProperty(put_gcs_object_, PutGCSObject::ObjectACL, magic_enum::enum_name(minifi_gcp::put_gcs_object::PredefinedAcl::AUTHENTICATED_READ))); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Bucket.name, "bucket-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::Key.name, "object-name-from-property")); + EXPECT_TRUE(test_controller_.getProcessor()->setProperty(PutGCSObject::ObjectACL.name, + std::string(magic_enum::enum_name(minifi_gcp::put_gcs_object::PredefinedAcl::AUTHENTICATED_READ)))); const auto& result = test_controller_.trigger("hello world"); ASSERT_EQ(1, result.at(PutGCSObject::Success).size()); EXPECT_EQ(0, result.at(PutGCSObject::Failure).size()); diff --git a/extensions/llamacpp/CMakeLists.txt b/extensions/llamacpp/CMakeLists.txt index 1de6be4589..d8fd187a21 100644 --- a/extensions/llamacpp/CMakeLists.txt +++ b/extensions/llamacpp/CMakeLists.txt @@ -25,7 +25,7 @@ include(LlamaCpp) include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) -file(GLOB SOURCES "processors/*.cpp") +file(GLOB SOURCES "processors/*.cpp" "ExtensionInitializer.cpp") add_minifi_library(minifi-llamacpp SHARED ${SOURCES}) target_include_directories(minifi-llamacpp PUBLIC "${CMAKE_SOURCE_DIR}/extensions/llamacpp") diff --git a/extensions/llamacpp/processors/ExtensionInitializer.cpp b/extensions/llamacpp/ExtensionInitializer.cpp similarity index 76% rename from extensions/llamacpp/processors/ExtensionInitializer.cpp rename to extensions/llamacpp/ExtensionInitializer.cpp index 85d610a7d9..09b142ff21 100644 --- a/extensions/llamacpp/processors/ExtensionInitializer.cpp +++ b/extensions/llamacpp/ExtensionInitializer.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include "RunLlamaCppInference.h" #include "api/core/Resource.h" #include "api/utils/minifi-c-utils.h" +#include "processors/RunLlamaCppInference.h" #define MKSOC(x) #x #define MAKESTRING(x) MKSOC(x) // NOLINT(cppcoreguidelines-macro-usage) @@ -28,13 +28,11 @@ CEXTENSIONAPI const uint32_t MinifiApiVersion = MINIFI_API_VERSION; CEXTENSIONAPI void MinifiInitExtension(MinifiExtensionContext* extension_context) { MinifiExtensionDefinition extension_definition{ - .name = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_NAME)), - .version = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_VERSION)), + .name = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_NAME)), + .version = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_VERSION)), .deinit = nullptr, .user_data = nullptr }; auto* extension = MinifiRegisterExtension(extension_context, &extension_definition); - minifi::api::core::useProcessorClassDefinition([&] (const MinifiProcessorClassDefinition& definition) { - MinifiRegisterProcessor(extension, &definition); - }); + minifi::api::core::registerProcessors(extension); } diff --git a/extensions/llamacpp/processors/RunLlamaCppInference.cpp b/extensions/llamacpp/processors/RunLlamaCppInference.cpp index ede77cd062..888280cb1c 100644 --- a/extensions/llamacpp/processors/RunLlamaCppInference.cpp +++ b/extensions/llamacpp/processors/RunLlamaCppInference.cpp @@ -32,7 +32,7 @@ MinifiStatus RunLlamaCppInference::onScheduleImpl(api::core::ProcessContext& con model_path_.clear(); model_path_ = api::utils::parseProperty(context, ModelPath); multimodal_model_path_ = api::utils::parseOptionalProperty(context, MultiModalModelPath); - system_prompt_ = context.getProperty(SystemPrompt).value_or(""); + system_prompt_ = api::utils::parseOptionalProperty(context, SystemPrompt).value_or(""); output_attribute_ = api::utils::parseOptionalProperty(context, OutputAttributeName); LlamaSamplerParams llama_sampler_params; diff --git a/libminifi/src/minifi-c.cpp b/libminifi/src/minifi-c.cpp index 1c5d98fcf0..ea50f1bf7a 100644 --- a/libminifi/src/minifi-c.cpp +++ b/libminifi/src/minifi-c.cpp @@ -25,6 +25,7 @@ #include "core/extension/ExtensionManager.h" #include "minifi-cpp/Exception.h" #include "minifi-cpp/controllers/SSLContextServiceInterface.h" +#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" #include "minifi-cpp/core/Annotation.h" #include "minifi-cpp/core/ClassLoader.h" #include "minifi-cpp/core/ProcessContext.h" @@ -180,6 +181,16 @@ class CControllerServiceFactory : public minifi::core::controller::ControllerSer minifi::utils::CControllerServiceClassDescription class_description_; }; +MinifiProxyType minifiProxyType(const minifi::controllers::ProxyType& proxy_type) { + switch (proxy_type) { + case minifi::controllers::ProxyType::DIRECT: + return MinifiProxyType::MINIFI_PROXY_TYPE_DIRECT; + case minifi::controllers::ProxyType::HTTP: + return MinifiProxyType::MINIFI_PROXY_TYPE_HTTP; + } + std::unreachable(); +} + } // namespace namespace org::apache::nifi::minifi::utils { @@ -578,9 +589,9 @@ MinifiStatus MinifiControllerServiceContextGetProperty(MinifiControllerServiceCo } } -MinifiStatus MinifiProcessContextGetControllerService( +MinifiStatus MinifiProcessContextGetControllerServiceFromProperty( MinifiProcessContext* process_context, - const MinifiStringView controller_service_name, + const MinifiStringView property_name, const MinifiStringView controller_service_type, MinifiControllerService** controller_service_out) { if (!controller_service_out) { @@ -589,8 +600,10 @@ MinifiStatus MinifiProcessContextGetControllerService( gsl_Assert(process_context != MINIFI_NULL); const auto context = reinterpret_cast(process_context); - const auto name_str = std::string{toStringView(controller_service_name)}; - const auto service_shared_ptr = context->getControllerService(name_str, context->getProcessorInfo().getUUID()); + const auto property_name_str = std::string{toStringView(property_name)}; + const auto name_str = context->getProperty(property_name_str, nullptr); + if (!name_str) { return MINIFI_STATUS_PROPERTY_NOT_SET; } + const auto service_shared_ptr = context->getControllerService(*name_str, context->getProcessorInfo().getUUID()); if (!service_shared_ptr) { return MINIFI_STATUS_VALIDATION_FAILED; } @@ -643,4 +656,32 @@ MinifiStatus MinifiProcessContextGetSslData(MinifiProcessContext* process_contex } +MinifiStatus MinifiProcessContextGetProxyDataFromProperty(MinifiProcessContext* process_context, MinifiStringView property_name, + void (*cb)(void* user_ctx, const MinifiProxyData* proxy_data), void* user_ctx) { + gsl_Assert(process_context != MINIFI_NULL); + const auto context = reinterpret_cast(process_context); + const auto property_name_str = std::string{toStringView(property_name)}; + const auto name_str = context->getProperty(property_name_str, nullptr); + if (!name_str) { return MINIFI_STATUS_PROPERTY_NOT_SET; } + const auto service_shared_ptr = context->getControllerService(*name_str, context->getProcessorInfo().getUUID()); + if (!service_shared_ptr) { return MINIFI_STATUS_VALIDATION_FAILED; } + if (const auto proxy_service = dynamic_cast(service_shared_ptr.get())) { + const std::string hostname = proxy_service->getHost(); + const auto basic_auth_data = proxy_service->getProxyCredentials(); + MinifiStringView username_holder = basic_auth_data ? minifiStringView(basic_auth_data->username) : MinifiStringView{}; + MinifiStringView password_holder = basic_auth_data ? minifiStringView(basic_auth_data->password) : MinifiStringView{}; + + MinifiProxyData proxy_data{ + .hostname = minifiStringView(hostname), + .port = proxy_service->getPort(), + .username = basic_auth_data ? &username_holder : nullptr, + .password = basic_auth_data ? &password_holder : nullptr, + .proxy_type = minifiProxyType(proxy_service->getProxyType()), + }; + cb(user_ctx, &proxy_data); + return MINIFI_STATUS_SUCCESS; + } + return MINIFI_STATUS_VALIDATION_FAILED; +} + } // extern "C" diff --git a/libminifi/test/integration/extension-verification-test/CApiExtension.cpp b/libminifi/test/integration/extension-verification-test/CApiExtension.cpp index 546a229056..13bec7e0c6 100644 --- a/libminifi/test/integration/extension-verification-test/CApiExtension.cpp +++ b/libminifi/test/integration/extension-verification-test/CApiExtension.cpp @@ -44,8 +44,8 @@ CEXTENSIONAPI const uint32_t MinifiApiVersion = MINIFI_TEST_API_VERSION; CEXTENSIONAPI void MinifiInitExtension(MinifiExtensionContext* extension_context) { MinifiExtensionDefinition extension_definition{ - .name = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_NAME)), - .version = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_VERSION)), + .name = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_NAME)), + .version = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_VERSION)), .deinit = nullptr, .user_data = nullptr }; diff --git a/minifi-api/include/minifi-c/minifi-c.h b/minifi-api/include/minifi-c/minifi-c.h index e846efbdcf..b54375670f 100644 --- a/minifi-api/include/minifi-c/minifi-c.h +++ b/minifi-api/include/minifi-c/minifi-c.h @@ -45,6 +45,11 @@ extern "C" { /// use MINIFI_SSL_CONTEXT_SERVICE_PROPERTY_TYPE in the type field of the property definition (MinifiPropertyDefinition::type) #define MINIFI_SSL_CONTEXT_SERVICE_PROPERTY_TYPE "org.apache.nifi.minifi.controllers.SSLContextServiceInterface" +/// To declare a processor property that expects an ProxyConfigurationService, +/// use MINIFI_PROXY_CONFIGURATION_SERVICE_PROPERTY_TYPE in the type field of the property definition (MinifiPropertyDefinition::type) +#define MINIFI_PROXY_CONFIGURATION_SERVICE_PROPERTY_TYPE "org.apache.nifi.minifi.controllers.ProxyConfigurationServiceInterface" + + enum : uint32_t { MINIFI_API_VERSION = 3 }; @@ -233,8 +238,8 @@ MinifiStatus MinifiProcessContextGetProperty(MinifiProcessContext* context, Mini void(*cb)(void* user_ctx, MinifiStringView property_value), void* user_ctx); MinifiBool MinifiProcessContextHasNonEmptyProperty(MinifiProcessContext* context, MinifiStringView property_name); -MinifiStatus MinifiProcessContextGetControllerService( - MinifiProcessContext* process_context, MinifiStringView controller_service_name, MinifiStringView controller_service_type, MinifiControllerService** controller_service_out); +MinifiStatus MinifiProcessContextGetControllerServiceFromProperty( + MinifiProcessContext* process_context, MinifiStringView property_name, MinifiStringView controller_service_type, MinifiControllerService** controller_service_out); void MinifiProcessContextGetDynamicProperties(MinifiProcessContext* context, MinifiFlowFile* minifi_flow_file, void (*cb)(void* user_ctx, MinifiStringView dynamic_property_name, MinifiStringView dynamic_property_value), void* user_ctx); @@ -283,6 +288,22 @@ typedef struct MinifiSslData { MinifiStatus MinifiProcessContextGetSslData(MinifiProcessContext* process_context, MinifiStringView controller_service_name, void (*cb)(void* user_ctx, const MinifiSslData* ssl_data), void* user_ctx); +typedef enum MinifiProxyType : uint8_t { + MINIFI_PROXY_TYPE_DIRECT, + MINIFI_PROXY_TYPE_HTTP +} MinifiProxyType; + +typedef struct MinifiProxyData { + MinifiStringView hostname; + uint16_t port; + MinifiStringView* username; + MinifiStringView* password; + MinifiProxyType proxy_type; +} MinifiProxyData; + +MinifiStatus MinifiProcessContextGetProxyDataFromProperty(MinifiProcessContext* process_context, MinifiStringView property_name, + void (*cb)(void* user_ctx, const MinifiProxyData* proxy_data), void* user_ctx); + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/minifi-api/minifi-c-api.def b/minifi-api/minifi-c-api.def index 0fcaf2ad97..24b65765e2 100644 --- a/minifi-api/minifi-c-api.def +++ b/minifi-api/minifi-c-api.def @@ -4,7 +4,7 @@ EXPORTS MinifiRegisterProcessor MinifiRegisterControllerService MinifiPublishedMetricsCreate - MinifiProcessContextGetControllerService + MinifiProcessContextGetControllerServiceFromProperty MinifiProcessContextGetProperty MinifiProcessContextHasNonEmptyProperty MinifiControllerServiceContextGetProperty @@ -30,3 +30,4 @@ EXPORTS MinifiProcessSessionGetFlowFileId MinifiProcessContextGetDynamicProperties MinifiProcessContextGetSslData + MinifiProcessContextGetProxyDataFromProperty