From 44ae877db72a62b77d9403ac7ab3ed3cf3e74d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20St=C3=B6ckli?= Date: Fri, 12 Jun 2026 11:36:49 +0200 Subject: [PATCH 1/2] feat: add taskflow prompts to handle libraries in addition to applications, biggest thanks to @m-y-mo --- scripts/audit/run_audit.sh | 22 +- .../mcp_servers/local_gh_resources.py | 20 ++ .../mcp_servers/repo_context.py | 207 ++++++++++++++- .../mcp_servers/repo_context_models.py | 44 ++++ .../prompts/audit/audit_library_issue.yaml | 92 +++++++ .../audit/classify_application_component.yaml | 42 +++ .../audit/classify_library_component.yaml | 20 ++ .../audit/entry_point_application.yaml | 17 ++ .../prompts/audit/entry_point_library.yaml | 30 +++ .../audit/record_library_dependency.yaml | 22 ++ .../audit/user_action_application.yaml | 15 ++ .../prompts/audit/user_action_library.yaml | 19 ++ .../audit/audit_issue_local_iter.yaml | 8 +- .../audit/audit_library_issue_local.yaml | 53 ++++ .../audit/classify_application_local.yaml | 87 ++++--- .../classify_library_application_local.yaml | 145 +++++++++++ .../gather_security_entry_point_info.yaml | 35 +++ .../audit/identify_applications.yaml | 82 ++---- .../audit/identify_library_components.yaml | 241 ++++++++++++++++++ 19 files changed, 1092 insertions(+), 109 deletions(-) create mode 100644 src/seclab_taskflows/prompts/audit/audit_library_issue.yaml create mode 100644 src/seclab_taskflows/prompts/audit/classify_application_component.yaml create mode 100644 src/seclab_taskflows/prompts/audit/classify_library_component.yaml create mode 100644 src/seclab_taskflows/prompts/audit/entry_point_application.yaml create mode 100644 src/seclab_taskflows/prompts/audit/entry_point_library.yaml create mode 100644 src/seclab_taskflows/prompts/audit/record_library_dependency.yaml create mode 100644 src/seclab_taskflows/prompts/audit/user_action_application.yaml create mode 100644 src/seclab_taskflows/prompts/audit/user_action_library.yaml create mode 100644 src/seclab_taskflows/taskflows/audit/audit_library_issue_local.yaml create mode 100644 src/seclab_taskflows/taskflows/audit/classify_library_application_local.yaml create mode 100644 src/seclab_taskflows/taskflows/audit/gather_security_entry_point_info.yaml create mode 100644 src/seclab_taskflows/taskflows/audit/identify_library_components.yaml diff --git a/scripts/audit/run_audit.sh b/scripts/audit/run_audit.sh index 23aaae2..14b3bb8 100755 --- a/scripts/audit/run_audit.sh +++ b/scripts/audit/run_audit.sh @@ -5,14 +5,19 @@ set -e USE_ADVISORY=false +MODEL_CONFIG_FLAG="" # Parse flags -while [[ "$1" == --* ]]; do +while [[ "$1" == -* ]]; do case "$1" in --advisory) USE_ADVISORY=true shift ;; + -m) + MODEL_CONFIG_FLAG="-m $2" + shift 2 + ;; *) echo "Unknown option: $1" exit 1 @@ -21,20 +26,21 @@ while [[ "$1" == --* ]]; do done if [ -z "$1" ]; then - echo "Usage: $0 [--advisory] "; + echo "Usage: $0 [--advisory] [-m model_config] "; exit 1; fi -python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.fetch_source_code -g repo="$1" -python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.identify_applications -g repo="$1" -python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.gather_web_entry_point_info -g repo="$1" +python -m seclab_taskflow_agent $MODEL_CONFIG_FLAG -t seclab_taskflows.taskflows.audit.fetch_source_code -g repo="$1" +python -m seclab_taskflow_agent $MODEL_CONFIG_FLAG -t seclab_taskflows.taskflows.audit.identify_applications -g repo="$1" +python -m seclab_taskflow_agent $MODEL_CONFIG_FLAG -t seclab_taskflows.taskflows.audit.gather_web_entry_point_info -g repo="$1" +python -m seclab_taskflow_agent $MODEL_CONFIG_FLAG -t seclab_taskflows.taskflows.audit.gather_security_entry_point_info -g repo="$1" if [ "$USE_ADVISORY" = true ]; then - python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.fetch_security_advisories -g repo="$1" + python -m seclab_taskflow_agent $MODEL_CONFIG_FLAG -t seclab_taskflows.taskflows.audit.fetch_security_advisories -g repo="$1" fi -python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.classify_application_local -g repo="$1" -g use_advisory="$USE_ADVISORY" -python -m seclab_taskflow_agent -t seclab_taskflows.taskflows.audit.audit_issue_local_iter -g repo="$1" -g use_advisory="$USE_ADVISORY" +python -m seclab_taskflow_agent $MODEL_CONFIG_FLAG -t seclab_taskflows.taskflows.audit.classify_application_local -g repo="$1" -g use_advisory="$USE_ADVISORY" +python -m seclab_taskflow_agent $MODEL_CONFIG_FLAG -t seclab_taskflows.taskflows.audit.audit_issue_local_iter -g repo="$1" -g use_advisory="$USE_ADVISORY" set +e diff --git a/src/seclab_taskflows/mcp_servers/local_gh_resources.py b/src/seclab_taskflows/mcp_servers/local_gh_resources.py index 44333f5..2d61d24 100644 --- a/src/seclab_taskflows/mcp_servers/local_gh_resources.py +++ b/src/seclab_taskflows/mcp_servers/local_gh_resources.py @@ -71,6 +71,18 @@ async def _fetch_file(url, headers, params): return await _fetch_file(url, headers=headers, params=params) +def _parse_content_disposition_filename(header: str | None) -> str | None: + """Extract the filename from a Content-Disposition header value.""" + if not header: + return None + for part in header.split(";"): + part = part.strip() + if part.lower().startswith("filename="): + filename = part.split("=", 1)[1].strip().strip('"') + return filename + return None + + async def _fetch_source_zip(owner: str, repo: str, tmp_dir): """Fetch the source code.""" url = f"https://api.github.com/repos/{owner}/{repo}/zipball" @@ -83,6 +95,8 @@ async def _fetch_source_zip(owner: str, repo: str, tmp_dir): async with httpx.AsyncClient() as client: async with client.stream("GET", url, headers=headers, follow_redirects=True) as response: response.raise_for_status() + content_disposition = response.headers.get("content-disposition") + source_filename = _parse_content_disposition_filename(content_disposition) expected_path = Path(tmp_dir) / owner / f"{repo}.zip" resolved_path = expected_path.resolve() if os.path.commonpath([resolved_path, Path(tmp_dir).resolve()]) != str(Path(tmp_dir).resolve()): @@ -92,6 +106,11 @@ async def _fetch_source_zip(owner: str, repo: str, tmp_dir): async with aiofiles.open(f"{tmp_dir}/{owner}/{repo}.zip", "wb") as f: async for chunk in response.aiter_bytes(): await f.write(chunk) + metadata = {"source_filename": source_filename} + metadata_path = Path(tmp_dir) / owner / f"{repo}_source_metadata.json" + async with aiofiles.open(metadata_path, "w") as f: + await f.write(json.dumps(metadata, indent=2)) + return f"source code for {repo} fetched successfully." except httpx.RequestError as e: return f"Error: Request error: {e}" @@ -113,6 +132,7 @@ async def fetch_repo_from_gh(owner: str, repo: str): source_path = Path(f"{LOCAL_GH_DIR}/{owner}/{repo}.zip") if not source_path.exists(): return result + return f"Downloaded source code to {owner}/{repo}.zip" diff --git a/src/seclab_taskflows/mcp_servers/repo_context.py b/src/seclab_taskflows/mcp_servers/repo_context.py index 67b9935..a5e8622 100644 --- a/src/seclab_taskflows/mcp_servers/repo_context.py +++ b/src/seclab_taskflows/mcp_servers/repo_context.py @@ -14,8 +14,8 @@ from pathlib import Path from seclab_taskflow_agent.path_utils import mcp_data_dir, log_file_name -from .repo_context_models import Application, EntryPoint, UserAction, WebEntryPoint, ApplicationIssue, AuditResult, Base -from .repo_context_models import LowSeverityAuditResult +from .repo_context_models import Application, EntryPoint, UserAction, WebEntryPoint, ApplicationIssue, AuditResult, SecurityEntryPoint, Base +from .repo_context_models import LowSeverityAuditResult, AuditResultEvidence, ExternalDependency from .utils import process_repo logging.basicConfig( @@ -33,8 +33,8 @@ def app_to_dict(result): "repo": result.repo.lower(), "location": result.location, "notes": result.notes, - "is_app": result.is_app, - "is_library": result.is_library, + "is_app": True if result.is_app else False, + "is_library": True if result.is_library else False, } @@ -75,6 +75,16 @@ def web_entry_point_to_dict(wep): "notes": wep.notes, } +def security_entry_point_to_dict(sep): + return { + "id": sep.id, + "entry_point_id": sep.entry_point_id, + "type": sep.type, + "component": sep.component, + "untrusted_input": sep.untrusted_input, + "repo": sep.repo.lower(), + "notes": sep.notes, + } def audit_result_to_dict(res): return { @@ -108,6 +118,9 @@ def __init__(self, memcache_state_dir: str): ApplicationIssue.__table__, AuditResult.__table__, LowSeverityAuditResult.__table__, + SecurityEntryPoint.__table__, + AuditResultEvidence.__table__, + ExternalDependency.__table__, ], ) @@ -227,6 +240,32 @@ def store_new_web_entry_point( session.add(new_web_entry_point) session.commit() return f"Updated or added web entry point for entry_point_id {entry_point_id} in {repo}." + + def store_new_security_entry_point(self, repo, entry_point_id, type, component, untrusted_input, notes, update=False): + with Session(self.engine) as session: + existing = session.query(SecurityEntryPoint).filter_by(repo=repo, entry_point_id=entry_point_id).first() + if existing: + existing.notes += notes + if type: + existing.type = type + if component is not None: + existing.component = component + if untrusted_input: + existing.untrusted_input = untrusted_input + else: + if update: + return f"No security entry point exists at repo {repo} with entry_point_id {entry_point_id}." + new_security_entry_point = SecurityEntryPoint( + repo=repo, + entry_point_id=entry_point_id, + type=type, + component=component, + untrusted_input=untrusted_input, + notes=notes, + ) + session.add(new_security_entry_point) + session.commit() + return f"Updated or added security entry point for entry_point_id {entry_point_id} in {repo}." def store_new_user_action(self, repo, app_id, file, line, notes, update=False): with Session(self.engine) as session: @@ -256,6 +295,46 @@ def store_low_severity_reason(self, repo, component_id, result_id, reason): session.add(new_low_severity_result) session.commit() return f"Updated or added low severity result for {repo} and result id {result_id}" + + def store_audit_result_evidence(self, repo, component_id, result_id, has_evidence, evidence_notes): + with Session(self.engine) as session: + existing = session.query(AuditResultEvidence).filter_by(repo=repo, result_id=result_id).first() + if existing: + existing.evidence_notes += evidence_notes + existing.has_evidence = has_evidence + else: + new_evidence_result = AuditResultEvidence( + repo=repo, + component_id=component_id, + result_id=result_id, + has_evidence=has_evidence, + evidence_notes=evidence_notes, + ) + session.add(new_evidence_result) + session.commit() + return f"Updated or added audit result evidence for {repo} and result id {result_id}" + + def store_new_external_dependency(self, repo, app_id, name, function_used, file, line, notes, update=False): + with Session(self.engine) as session: + existing = session.query(ExternalDependency).filter_by(repo=repo, name=name, function_used=function_used, + file=file, line=line).first() + if existing: + existing.notes += notes + else: + if update: + return f"No external dependency exists at repo {repo}, name {name} and function {function_used}." + new_external_dependency = ExternalDependency( + repo=repo, + app_id=app_id, + name=name, + function_used=function_used, + file=file, + line=line, + notes=notes, + ) + session.add(new_external_dependency) + session.commit() + return f"Updated or added external dependency for {name} in {repo}." def get_app(self, repo, location): with Session(self.engine) as session: @@ -286,6 +365,8 @@ def get_app_issues(self, repo, component_id): "issue_type": issue.issue_type, "issue_notes": issue.notes, "issue_id": issue.id, + "is_library": True if app.is_library else False, + "is_app": True if app.is_app else False, } for app, issue in issues ] @@ -359,6 +440,21 @@ def get_web_entries_for_repo(self, repo): for r in results ] + def get_security_entries_for_repo(self, repo): + with Session(self.engine) as session: + results = session.query(SecurityEntryPoint).filter_by(repo=repo).all() + return [ + { + "repo": r.repo, + "entry_point_id": r.entry_point_id, + "type": r.type, + "component": r.component, + "untrusted_input": r.untrusted_input, + "notes": r.notes, + } + for r in results + ] + def get_web_entries(self, repo, component_id): with Session(self.engine) as session: results = session.query(WebEntryPoint).filter_by(repo=repo, component=component_id).all() @@ -376,6 +472,21 @@ def get_web_entries(self, repo, component_id): } for r in results ] + + def get_security_entries(self, repo, component_id): + with Session(self.engine) as session: + results = session.query(SecurityEntryPoint).filter_by(repo=repo, component=component_id).all() + return [ + { + "repo": r.repo, + "entry_point_id": r.entry_point_id, + "type": r.type, + "component": r.component, + "untrusted_input": r.untrusted_input, + "notes": r.notes, + } + for r in results + ] def get_user_actions(self, repo, location): with Session(self.engine) as session: @@ -408,6 +519,8 @@ def clear_repo(self, repo): session.query(WebEntryPoint).filter_by(repo=repo).delete() session.query(AuditResult).filter_by(repo=repo).delete() session.query(LowSeverityAuditResult).filter_by(repo=repo).delete() + session.query(SecurityEntryPoint).filter_by(repo=repo).delete() + session.query(AuditResultEvidence).filter_by(repo=repo).delete() session.commit() return f"Cleared results for repo {repo}" @@ -517,7 +630,6 @@ def store_new_web_entry_point( owner: str = Field(description="The owner of the GitHub repository"), repo: str = Field(description="The name of the GitHub repository"), entry_point_id: int = Field(description="The ID of the entry point this web entry point refers to"), - location: str = Field(description="The directory of the component where the web entry point belongs to"), method: str = Field(description="HTTP method (GET, POST, etc)", default=""), path: str = Field(description="URL path (e.g., /info)", default=""), component: int = Field(description="Component identifier", default=0), @@ -534,6 +646,23 @@ def store_new_web_entry_point( process_repo(owner, repo), entry_point_id, method, path, component, auth, middleware, roles_scopes, notes ) +@mcp.tool() +def store_new_security_entry_point( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + entry_point_id: int = Field(description="The ID of the entry point this security entry point refers to"), + type: str = Field(description="The type of the security entry point, e.g., SQL injection, XSS, etc.", default=""), + component: int = Field(description="Component identifier", default=0), + untrusted_input: str = Field(description="Untrusted input information", default=""), + notes: str = Field(description="Notes for this security entry point", default=""), +): + """ + Stores a new security entry point in a component to the database. A security entry point extends a regular entry point + with security-specific properties like type of vulnerability and untrusted input information. + """ + return backend.store_new_security_entry_point( + process_repo(owner, repo), entry_point_id, type, component, untrusted_input, notes + ) @mcp.tool() def add_entry_point_notes( @@ -588,6 +717,23 @@ def add_user_action_notes( return f"Error: No component exists in repo: {repo} and location {location}" return backend.store_new_user_action(repo, app.id, file, line, notes, True) +@mcp.tool() +def store_new_external_dependency( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + component_id: int = Field(description="The ID of the component where the external dependency belongs to"), + name: str = Field(description="The name of the external dependency"), + function_used: str = Field(description="The function used from the external dependency"), + file: str = Field(description="The file that contains the external dependency"), + line: int = Field(description="The file line that contains the external dependency"), + notes: str = Field(description="New notes for this external dependency", default=""), +): + """ + Stores a new external dependency in a component to the database. + """ + repo = process_repo(owner, repo) + return backend.store_new_external_dependency(repo, component_id, name, function_used, file, line, notes) + @mcp.tool() def get_component( @@ -616,6 +762,21 @@ def get_components( repo = process_repo(owner, repo) return json.dumps(backend.get_apps(repo)) +@mcp.tool() +def get_component_by_id( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + component_id: int = Field(description="The ID of the component"), +): + """ + Get a component by id. + """ + repo = process_repo(owner, repo) + with Session(backend.engine) as session: + app = session.query(Application).filter_by(repo=repo, id=component_id).first() + if not app: + return f"Error: No component exists in repo: {repo} with id {component_id}" + return json.dumps(app_to_dict(app)) @mcp.tool() def get_entry_points( @@ -654,6 +815,17 @@ def get_web_entry_points_component( repo = process_repo(owner, repo) return json.dumps(backend.get_web_entries(repo, component_id)) +@mcp.tool() +def get_security_entry_points_component( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + component_id: int = Field(description="The ID of the component"), +): + """ + Get all security entry points for a component + """ + repo = process_repo(owner, repo) + return json.dumps(backend.get_security_entries(repo, component_id)) @mcp.tool() def get_web_entry_points_for_repo( @@ -666,6 +838,16 @@ def get_web_entry_points_for_repo( repo = process_repo(owner, repo) return json.dumps(backend.get_web_entries_for_repo(repo)) +@mcp.tool() +def get_security_entry_points_for_repo( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), +): + """ + Get all security entry points of an repo + """ + repo = process_repo(owner, repo) + return json.dumps(backend.get_security_entries_for_repo(repo)) @mcp.tool() def get_user_actions( @@ -815,6 +997,21 @@ def store_low_severity_reason( repo = process_repo(owner, repo) return backend.store_low_severity_reason(repo, component_id, result_id, reason) +@mcp.tool() +def store_audit_result_evidence( + owner: str = Field(description="The owner of the GitHub repository"), + repo: str = Field(description="The name of the GitHub repository"), + component_id: int = Field(description="The ID of the component"), + result_id: int = Field(description="The ID of the audit result"), + has_evidence: bool = Field(description="Whether there is evidence for this audit result"), + evidence_notes: str = Field(description="Notes about the evidence for this audit result"), +): + """ + Store evidence information for an audit result. + """ + repo = process_repo(owner, repo) + return backend.store_audit_result_evidence(repo, component_id, result_id, has_evidence, evidence_notes) + @mcp.tool() def clear_repo( owner: str = Field(description="The owner of the GitHub repository"), diff --git a/src/seclab_taskflows/mcp_servers/repo_context_models.py b/src/seclab_taskflows/mcp_servers/repo_context_models.py index 6716849..b2e0047 100644 --- a/src/seclab_taskflows/mcp_servers/repo_context_models.py +++ b/src/seclab_taskflows/mcp_servers/repo_context_models.py @@ -74,6 +74,22 @@ def __repr__(self): f"component_id={self.component_id}, result_id={self.result_id}, reason={self.reason})>" ) +class AuditResultEvidence(Base): + __tablename__ = "audit_result_evidence" + id: Mapped[int] = mapped_column(primary_key=True) + repo: Mapped[str] + component_id = Column(Integer, ForeignKey("application.id", ondelete="CASCADE")) + result_id = Column(Integer, ForeignKey("audit_result.id", ondelete="CASCADE")) + has_evidence: Mapped[bool] + evidence_notes: Mapped[str] = mapped_column(Text) + + def __repr__(self): + return ( + f"" + ) + class EntryPoint(Base): __tablename__ = "entry_point" @@ -114,6 +130,23 @@ def __repr__(self): f"notes={self.notes}, repo={self.repo})>" ) +class SecurityEntryPoint(Base): + __tablename__ = "security_entry_point" + + id: Mapped[int] = mapped_column(primary_key=True) + entry_point_id = Column(Integer, ForeignKey("entry_point.id", ondelete="CASCADE")) + type: Mapped[str] + component: Mapped[int] + untrusted_input: Mapped[str] + notes: Mapped[str] = mapped_column(Text) + repo: Mapped[str] + + def __repr__(self): + return ( + f"" + ) class UserAction(Base): __tablename__ = "user_action" @@ -124,3 +157,14 @@ class UserAction(Base): file: Mapped[str] line: Mapped[int] notes: Mapped[str] = mapped_column(Text) + +class ExternalDependency(Base): + __tablename__ = "external_dependency" + id: Mapped[int] = mapped_column(primary_key=True) + repo: Mapped[str] + app_id = Column(Integer, ForeignKey("application.id", ondelete="CASCADE")) + name: Mapped[str] + function_used: Mapped[str] + file: Mapped[str] + line: Mapped[str] + notes: Mapped[str] = mapped_column(Text) diff --git a/src/seclab_taskflows/prompts/audit/audit_library_issue.yaml b/src/seclab_taskflows/prompts/audit/audit_library_issue.yaml new file mode 100644 index 0000000..7543855 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit/audit_library_issue.yaml @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: prompt + version: "1.0" +prompt: | + The issues suggested have not been properly verified and are only suggested because they are common issues in these types of + application. Your task is to audit the source code to check if this type of issues is present. + + Note that the issues are meant to be suggestions of the risk present in the component and the file paths included are + used as examples and not exhaustive. You should take the general idea of the risk and audit the entire component, going + beyond the files and examples suggested in the issue. + + You need to take into account of the intention and threat model of the component in component notes to determine if an issue + is a valid security issue or if it is an intended functionality. You can fetch entry points, web entry points and user actions + to help you determine the intended usage of the component. + + For example, the application may be a dashboard frontend that allow user to query various databases, in this case, being able + to query these databases is not considered a security issue or info leak. + + Or an application may be an Admin UI intended for an admin user to perform high privilege actions. In this case, the absence of + authentication or bypasses would be a security issue. However, issues that are reachable only after authentication are not + generally considered security issue, unless it has unintended consequences or are XSS, CSRF type issues. + + Remember to take intended usage into account. For example, a CLI tool may be capable of high privilege actions, but it may + only be intended to be used by trusted users for local testing etc. and any actions only affects the local environment of the + user. In this case, inputs to the CLI is not in general considered as untrusted input, and being able to perform high privilege + actions with the CLI is generally not a vulnerability. + + For library components, it may not always be clear how the downstream code is using the library and whether the code is + consuming untrusted input. In this case, + you should look at the notes, and the documentation or README of the repository to determine the likely usage of the library. + In the case of library components, a realistic exploitable sceanrio is when a typical application using the library + becomes exploitable due to the issue in the library. + + For example, a function may be processing a URL and it is not clear if downstream code is likely to pass untrusted URL + to it. In this case, you can look for code examples in the repo, or documentations to see what is the typical usage of + the function. The documentation and function name may also give you an idea of the intented usage of the function, for example, + are the users likely to expect the function to be able to process unsanitized input? Is this function meant to + sanitize untrusted input? Is there documentation, or configurations that warns users to not pass untrusted input to the function? + + Another example is when the library acquires unintended privilege with normal usage, such as when an library accepting ldap + query, but does not properly sanitize the query, which then leads to remote code execution, or when a memory corrupition issue + in an image parsing library that may be exploited to achieve code execution while parsing an image. + + When considering attack scenarios in library components, you should show how an application that uses the library can be exploited due to the issue in the library. + While the application that demonstrates the attack should be simple and use for testing only, you should still make sure that + the application represents a realistic usage of the library. + + For example, if the function is intended to be a progress bar processing async events, and a vulnerable application needs to + use it for processing static files or outputs, then it is not a realistic scenario, + because the library is not likely to be used this way. + + You should also ensure that it is not deliberately designed or configured to be vulnerable. + Where the library provides configurations that may mitigate the issue, you should assume that these configurations are properly set. + + Do not consider scenarios where authentication is bypassed via stolen credential etc. We only consider situations that are + achievable from within the source code itself. + + Keep a record of the audit notes, be sure to include all relevant file path and line number. Just stating an end point, e.g. + `IDOR in user update/delete endpoints (PUT /user/:id)` is not sufficient. I need to have the file and line number. + + If you believe there is a vulnerability, then you must include a realistic attack scenario, with details of all the file and line included, and also what an + attacker can gain by exploiting the vulnerability. Only consider the issue a vulnerability if an attacker can gain privilege by + performing an action that is not intended by the component. + + Remember, the issues suggested are only speculation and there may not be a vulnerability at all and it is ok to conclude that + there is no security issue. + + Likewise, if while auditing these issues, you noticed other inconsistencies that may lead to vulnerabilities according to the + criteria that I set out, then do investigate such issues and include them in the notes. + + Before reporting to the user, reflect on the notes and check the attack scenario: + 1. Have you included all the file and line number? + 2. Have you included a scenario of how the vulnerability can be exploited, including the end point and untrusted input that + may be used? + 3. Is this a realistic scenario that does not require stolen credential, misconfiguration etc. that is outside of what can be achieved via source code? + 4. In the attack scenario, what privilege can an attacker gain that is not intended by the application? + + Remember, if these criteria are not satisfied, it is ok to report that there is no vulnerabilities. + + In the end of the audit, collect your notes and store a new AuditResult entry. If the issue satisfies + the criteria for being a vulnerability, then set the `has_vulnerability` field to True when creating the + entry. If there are bugs that are due to incorrect implementation or inconsistencies, and it is not clear + whether it is exploitable or not, but nevertheless the bug should be addressed to harden security, then + set `has_non_security_error` to True but `has_vulnerability` to False. Set both to False if no security + related issues are identified. + + IMPORTANT + You must create a new AuditResult entry regardless of whether there is + any vulnerabilities or bugs. \ No newline at end of file diff --git a/src/seclab_taskflows/prompts/audit/classify_application_component.yaml b/src/seclab_taskflows/prompts/audit/classify_application_component.yaml new file mode 100644 index 0000000..e14138f --- /dev/null +++ b/src/seclab_taskflows/prompts/audit/classify_application_component.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: prompt + version: "1.0" +prompt: | + Identify the most likely security problems in the component. Your task is not to carry out a full audit, but to + identify the main risk in the component so that further analysis can be carried out. + Do not be too specific about an issue, but rather craft your report based on the general functionality and type of + the component and state the most likely risk, instead of risk specific to the implementation of the component. + However, you should still + take care not to include issues that are of low severity or require unrealistic attack scenarios such as misconfiguration + or an already compromised system. + + Base your decision on: + - Is this component likely to take untrusted user input? For example, remote web requests or IPC, RPC calls? + - What is the intended purpose of this component and its functionality? Does it allow high privileged actions? + Is it intended to provide such functionalities for all users? Or is there complex access control logic involved? + - The component itself may also have its own `README.md` (or a subdirectory of it may have a `README.md`). Take + a look at those files to help understand the functionality of the component. + + For example, an Admin UI/dashboard may be susceptible to client side Javascript vulnerabilities such as XSS, CSRF. + An authentication/authorization component may be susceptible to IDOR, bypasses etc. + Components that grant user access to data may require access control or authentication. It may also require + ownership checks. For example, + a web frontend may allow users to access their own content and admins to access all content, but users should not + be able to access another users' content in general. + + We're looking for more concrete and serious security issues that affects system integrity or + lead to information leak, so please do not include issues like brute force, Dos, log injection etc. + + Also do not include issues that require the system to be already compromised, such as issues that rely on malicious + configurations, stolen credentials, misconfiguration etc. + + For each scenario, consider what privilege and access the attacker needs to gain and do not include issues that + require high privilege or access to the system. + + Your task is to identify risk rather than properly audit and find security issues. Do not look too much into + the implementation or scrutinize the security measures such as access control and sanitizers at this stage. + Instead, report more general risks that are associated with the type of component + that you are looking at. diff --git a/src/seclab_taskflows/prompts/audit/classify_library_component.yaml b/src/seclab_taskflows/prompts/audit/classify_library_component.yaml new file mode 100644 index 0000000..d561de9 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit/classify_library_component.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: prompt + version: "1.0" +prompt: | + For each security entry point, identify the type of the security entry point and its functionality. + These entry points should be performing some kind of security sensitive operations, such as authentication, authorization, input validation, sanitization, escaping etc. + Identify the functionality of the entry point and what kind of issue may arise if the implementation is flawed. + For example, an authentication entry point may be susceptible to authentication bypass, + an input validation entry point may be susceptible to injection issues if the input is not properly sanitized, + an authorization entry point may be susceptible to IDOR or access control bypass if there are no proper access control checks. + This task is only to identify the potential risks and issues that may arise from the type of the security entry point, + rather than auditing the implementation of the entry point, so do not be too specific about the issue, and do not + audit the implementation, just identify the potential risks based on the type and functionality of the security entry point. + However, you should still take care not to include issues that are of low severity or require + unrealistic attack scenarios such as misconfiguration or an already compromised system, or issues that requires + attacker control of trusted inputs such as whitelist or blacklist bypass that requires the attacker + to be able to modify the whitelist or blacklist. \ No newline at end of file diff --git a/src/seclab_taskflows/prompts/audit/entry_point_application.yaml b/src/seclab_taskflows/prompts/audit/entry_point_application.yaml new file mode 100644 index 0000000..bff0de5 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit/entry_point_application.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: prompt + version: "1.0" +prompt: | + This component is an application component, you should consider the following: + + Does this application consume untrusted input, for example, web requests? Or does it interact with + other applications and can take input from them? User supplied arguments, + configuration files, web requests from hardcoded URLs or environment variables are not considered untrusted + input. + + However, if user arguments are pointing to some untrusted resources, then they can be considered as + untrusted input. For example, a parser may take a user argument that points to a file, and in this case, + the file is still considered untrusted even though it is supplied via user argument. diff --git a/src/seclab_taskflows/prompts/audit/entry_point_library.yaml b/src/seclab_taskflows/prompts/audit/entry_point_library.yaml new file mode 100644 index 0000000..1ac1848 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit/entry_point_library.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: prompt + version: "1.0" +prompt: | + This component is a library, identify the functions that are exposed to a library user. + Use the component notes, documentation, source code and examples to + identify these functions and determine how they are meant to be used and whether + they are meant to consume untrusted input. + + Consider the library user + as trusted and instead focus on functionalities that consume untrusted input when they are used for + building applications. + + For example, a web application framework may expose functionalities such as http request handlers and middleware, which + are typically used for handling http requests when used for building web applications. In this case, the + http requests handled by these handlers and middleware are considered untrusted. + + Sometimes it may not be clear whether a function is consuming untrusted input. In this case, + use hints from documentation, comments and usage examples to determine whether the + function is meant to consume untrusted input. + For example, a function may perform string formatting or processing URL, but the function + name and documentation strongly suggests that it is typically used for processing input + that comes from web requests. In this case, you can consider the function as consuming untrusted input. + + For library code, there may be functions that are not directly taking untrusted input, but are used as security + checks or sanitization. Make use of documentation, code comments, function names and usage examples to identify + these functions, and consider them to be entry points as well. \ No newline at end of file diff --git a/src/seclab_taskflows/prompts/audit/record_library_dependency.yaml b/src/seclab_taskflows/prompts/audit/record_library_dependency.yaml new file mode 100644 index 0000000..78bf6b7 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit/record_library_dependency.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: prompt + version: "1.0" +prompt: | + While auditing the issue, there may be functions from external library that are used for processing + an untrusted input or for security related operations such as validation. Note any such usage of external + library functions, except builtin library functions, and store a new external dependency entry in the database. + You should include the + function name, file and line. In the notes, you should specify how the library function is used, what untrusted + input is passed into the function and how a potential vulnerability in the function may impact the + application. For example, if the library function is responsible for validating a url, then a potential + vulnerability in the parsing logic for the function may lead to incorrect validation results, + potentially allowing malicious input to bypass security checks. + + Note that in some cases, a function that handles untrusted input or security-related operations may be invoked indirectly through other functions or callbacks, + and it may not be clear what function is called via the code path. In such cases, you need to examine the + dependencies used in the callback and identify potential libraries functions that may be used in the + these situations. You should include such function if it may be called in such context (handling untrusted input or + performing security related operations) diff --git a/src/seclab_taskflows/prompts/audit/user_action_application.yaml b/src/seclab_taskflows/prompts/audit/user_action_application.yaml new file mode 100644 index 0000000..ffe21c7 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit/user_action_application.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: prompt + version: "1.0" +prompt: | + This is an application components, you should consider the following: + + What actions can a user perform using the application? Can they exectue arbitrary code? Can they + make arbitrary web request? Can they read any local file? Consider restrictions that are placed on the actions + that a user can perform using the application, and what actions the user should not be allowed to do + when using the application. + + You should also state clearly the actions and privileges that a user can have when using the application. diff --git a/src/seclab_taskflows/prompts/audit/user_action_library.yaml b/src/seclab_taskflows/prompts/audit/user_action_library.yaml new file mode 100644 index 0000000..37431b7 --- /dev/null +++ b/src/seclab_taskflows/prompts/audit/user_action_library.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: prompt + version: "1.0" +prompt: | + This is a library component, do the following: + When considering user privileges, consider the library user as trusted and focus on the + functionalities that are exposed to untrusted users when the library is used. For example, a web + application library may provide functionalities for the library user to run arbitrary script to configure + the application. These are not considered "allowed actions" from an untrusted user point of view. + + However, if a library exposes a REST api that allows arbitrary files to be read by unauthenticated users, + then it reading arbitrary files is consider an "allowed" activity. Note any "allow" activity and how + they are exposed to untrusted users, referencing the file and function in detail. + + Ignore applications within the library such as default UI, etc. as these are going + to be analyzed separately. Focus on the library code. diff --git a/src/seclab_taskflows/taskflows/audit/audit_issue_local_iter.yaml b/src/seclab_taskflows/taskflows/audit/audit_issue_local_iter.yaml index 16ca7aa..fe3ddf7 100644 --- a/src/seclab_taskflows/taskflows/audit/audit_issue_local_iter.yaml +++ b/src/seclab_taskflows/taskflows/audit/audit_issue_local_iter.yaml @@ -23,7 +23,7 @@ taskflow: - seclab_taskflows.toolboxes.repo_context - task: must_complete: false - max_steps: 100 + max_steps: 300 repeat_prompt: true async: true agents: @@ -45,8 +45,14 @@ taskflow: {% if globals.use_advisory == 'true' %} {% include 'seclab_taskflows.prompts.audit.known_security_advisories' %} {% endif %} + + {% include 'seclab_taskflows.prompts.audit.record_library_dependency' %} + {% if result.is_library %} + {% include 'seclab_taskflows.prompts.audit.audit_library_issue' %} + {% else %} {% include 'seclab_taskflows.prompts.audit.audit_issue' %} + {% endif %} toolboxes: - seclab_taskflows.toolboxes.repo_context - seclab_taskflows.toolboxes.local_file_viewer diff --git a/src/seclab_taskflows/taskflows/audit/audit_library_issue_local.yaml b/src/seclab_taskflows/taskflows/audit/audit_library_issue_local.yaml new file mode 100644 index 0000000..a12d213 --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit/audit_library_issue_local.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" +model_config: seclab_taskflows.configs.model_config + +globals: + repo: + use_advisory: +# Taskflow to audit some potential issues. +taskflow: + - task: + must_complete: true + exclude_from_context: true + agents: + - seclab_taskflow_agent.personalities.assistant + model: general_tasks + user_prompt: | + Fetch all component issues for the repo {{ globals.repo }} using the get_component_issues_for_repo tool. + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - task: + must_complete: false + max_steps: 100 + repeat_prompt: true + async: true + agents: + - seclab_taskflows.personalities.web_application_security_expert + model: code_analysis + user_prompt: | + The issue is in repo {{ result.repo }} with id {{ result.issue_id }}. The component is under the directory + {{ result.location }} with component_id {{ result.component_id }}. The notes of the component is: + + {{ result.component_notes }} + + You should use this to understand the intended purpose of the component and take it into account when + you audit the issue. + + The type of the issue is {{ result.issue_type }} and here is the notes of the issue: + + {{ result.issue_notes }} + + {% if globals.use_advisory == 'true' %} + {% include 'seclab_taskflows.prompts.audit.known_security_advisories' %} + {% endif %} + + {% include 'seclab_taskflows.prompts.audit.audit_library_issue' %} + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.local_file_viewer + - seclab_taskflow_agent.toolboxes.memcache diff --git a/src/seclab_taskflows/taskflows/audit/classify_application_local.yaml b/src/seclab_taskflows/taskflows/audit/classify_application_local.yaml index aa9c926..507f651 100644 --- a/src/seclab_taskflows/taskflows/audit/classify_application_local.yaml +++ b/src/seclab_taskflows/taskflows/audit/classify_application_local.yaml @@ -43,50 +43,73 @@ taskflow: {% include 'seclab_taskflows.prompts.audit.known_security_advisories' %} {% endif %} - Identify the most likely security problems in the component. Your task is not to carry out a full audit, but to - identify the main risk in the component so that further analysis can be carried out. - Do not be too specific about an issue, but rather craft your report based on the general functionality and type of - the component and state the most likely risk, instead of risk specific to the implementation of the component. - However, you should still - take care not to include issues that are of low severity or require unrealistic attack scenarios such as misconfiguration - or an already compromised system. - - Base your decision on: - - Is this component likely to take untrusted user input? For example, remote web requests or IPC, RPC calls? - - What is the intended purpose of this component and its functionality? Does it allow high privileged actions? - Is it intended to provide such functionalities for all users? Or is there complex access control logic involved? - - The component itself may also have its own `README.md` (or a subdirectory of it may have a `README.md`). Take - a look at those files to help understand the functionality of the component. + {% if result.is_library %} + {% include 'seclab_taskflows.prompts.audit.classify_library_component' %} + {% else %} + {% include 'seclab_taskflows.prompts.audit.classify_application_component' %} + {% endif %} - For example, an Admin UI/dashboard may be susceptible to client side Javascript vulnerabilities such as XSS, CSRF. - An authentication/authorization component may be susceptible to IDOR, bypasses etc. - Components that grant user access to data may require access control or authentication. It may also require - ownership checks. For example, - a web frontend may allow users to access their own content and admins to access all content, but users should not - be able to access another users' content in general. + It is not your task to audit the security measures, but rather just to identify the risks and suggest some issues + that is worth auditing. - We're looking for more concrete and serious security issues that affects system integrity or - lead to information leak, so please do not include issues like brute force, Dos, log injection etc. + Reflect on your notes and check that the attack scenario meets the above requirements. Exclude low severity issues or + issues that require unrealistic circumstances such as misconfiguration or stolen credentials etc. + It is ok to conclude that the component is not likely to have security issues. - Also do not include issues that require the system to be already compromised, such as issues that rely on malicious - configurations, stolen credentials, misconfiguration etc. + If you think the issues satisfy the criteria, store a component issue entry for each type of issue identified. + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.local_file_viewer + - seclab_taskflows.toolboxes.ghsa + - task: + must_complete: true + exclude_from_context: true + agents: + - seclab_taskflow_agent.personalities.assistant + model: general_tasks + user_prompt: | + Fetch the components of the repo {{ globals.repo }} + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - task: + must_complete: true + repeat_prompt: true + async: true + max_steps: 100 + name: analyze_security_entry_points + agents: + - seclab_taskflows.personalities.web_application_security_expert + model: code_analysis + user_prompt: | + The component is in the repo {{ result.repo }} and it has component id {{ result.app_id }} and location {{ result.location }}. + The notes of the component is as follows. - For each scenario, consider what privilege and access the attacker needs to gain and do not include issues that - require high privilege or access to the system. + {{ result.notes }} - Your task is to identify risk rather than properly audit and find security issues. Do not look too much into - the implementation or scrutinize the security measures such as access control and sanitizers at this stage. - Instead, report more general risks that are associated with the type of component - that you are looking at. + Fetch the security entry points component, then the user actions of this component. + Based on the security entry points, components, user actions and README.md and if available, SECURITY.md in the {{ globals.repo }}, + can you tell me what type of application this repo is and what kind of security boundary it has. - It is not your task to audit the security measures, but rather just to identify the risks and suggest some issues - that is worth auditing. + For each security entry point, identify the type of the security entry point and its functionality. + These entry points should be performing some kind of security sensitive operations, such as authentication, authorization, input validation, sanitization, escaping etc. + Identify the functionality of the entry point and what kind of issue may arise if the implementation is flawed. + For example, an authentication entry point may be susceptible to authentication bypass, + an input validation entry point may be susceptible to injection issues if the input is not properly sanitized, + an authorization entry point may be susceptible to IDOR or access control bypass if there are no proper access control checks. + This task is only to identify the potential risks and issues that may arise from the type of the security entry point, + rather than auditing the implementation of the entry point, so do not be too specific about the issue, and do not + audit the implementation, just identify the potential risks based on the type and functionality of the security entry point. + However, you should still take care not to include issues that are of low severity or require + unrealistic attack scenarios such as misconfiguration or an already compromised system, or issues that requires + attacker control of trusted inputs such as whitelist or blacklist bypass that requires the attacker + to be able to modify the whitelist or blacklist. Reflect on your notes and check that the attack scenario meets the above requirements. Exclude low severity issues or issues that require unrealistic circumstances such as misconfiguration or stolen credentials etc. It is ok to conclude that the component is not likely to have security issues. If you think the issues satisfy the criteria, store a component issue entry for each type of issue identified. + toolboxes: - seclab_taskflows.toolboxes.repo_context - seclab_taskflows.toolboxes.local_file_viewer diff --git a/src/seclab_taskflows/taskflows/audit/classify_library_application_local.yaml b/src/seclab_taskflows/taskflows/audit/classify_library_application_local.yaml new file mode 100644 index 0000000..e0f1c3e --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit/classify_library_application_local.yaml @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" +model_config: seclab_taskflows.configs.model_config + +globals: + repo: + use_advisory: +# Taskflow to analyze the general contextual information of a project and classify the different applications within it +taskflow: + - task: + must_complete: true + exclude_from_context: true + agents: + - seclab_taskflow_agent.personalities.assistant + model: general_tasks + user_prompt: | + Fetch the components of the repo {{ globals.repo }} + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - task: + must_complete: true + repeat_prompt: true + async: true + agents: + - seclab_taskflows.personalities.web_application_security_expert + model: code_analysis + user_prompt: | + The component is in the repo {{ result.repo }} and it has component id {{ result.app_id }} and location {{ result.location }}. + The notes of the component is as follows. + + {{ result.notes }} + + Fetch the entry points, web entry points of the component, then the user actions of this component. + Based on the entry points, web entry points, components, user actions and README.md and if available, SECURITY.md in the {{ globals.repo }}, + can you tell me what type of application this repo is and what kind of security boundary it has. + Based on this, determine whether the component is likely to have security problems. + + {% if globals.use_advisory == 'true' %} + {% include 'seclab_taskflows.prompts.audit.known_security_advisories' %} + {% endif %} + + Identify the most likely security problems in the component. Your task is not to carry out a full audit, but to + identify the main risk in the component so that further analysis can be carried out. + Do not be too specific about an issue, but rather craft your report based on the general functionality and type of + the component and state the most likely risk, instead of risk specific to the implementation of the component. + However, you should still + take care not to include issues that are of low severity or require unrealistic attack scenarios such as misconfiguration + or an already compromised system. + + Base your decision on: + - Is this component likely to take untrusted user input? For example, remote web requests or IPC, RPC calls? + - What is the intended purpose of this component and its functionality? Does it allow high privileged actions? + Is it intended to provide such functionalities for all users? Or is there complex access control logic involved? + - The component itself may also have its own `README.md` (or a subdirectory of it may have a `README.md`). Take + a look at those files to help understand the functionality of the component. + + For example, an Admin UI/dashboard may be susceptible to client side Javascript vulnerabilities such as XSS, CSRF. + An authentication/authorization component may be susceptible to IDOR, bypasses etc. + Components that grant user access to data may require access control or authentication. It may also require + ownership checks. For example, + a web frontend may allow users to access their own content and admins to access all content, but users should not + be able to access another users' content in general. + + We're looking for more concrete and serious security issues that affects system integrity or + lead to information leak, so please do not include issues like brute force, Dos, log injection etc. + + Also do not include issues that require the system to be already compromised, such as issues that rely on malicious + configurations, stolen credentials, misconfiguration etc. + + For each scenario, consider what privilege and access the attacker needs to gain and do not include issues that + require high privilege or access to the system. + + Your task is to identify risk rather than properly audit and find security issues. Do not look too much into + the implementation or scrutinize the security measures such as access control and sanitizers at this stage. + Instead, report more general risks that are associated with the type of component + that you are looking at. + + It is not your task to audit the security measures, but rather just to identify the risks and suggest some issues + that is worth auditing. + + Reflect on your notes and check that the attack scenario meets the above requirements. Exclude low severity issues or + issues that require unrealistic circumstances such as misconfiguration or stolen credentials etc. + It is ok to conclude that the component is not likely to have security issues. + + If you think the issues satisfy the criteria, store a component issue entry for each type of issue identified. + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.local_file_viewer + - seclab_taskflow_agent.toolboxes.memcache + - task: + must_complete: true + exclude_from_context: true + agents: + - seclab_taskflow_agent.personalities.assistant + model: general_tasks + user_prompt: | + Fetch the components of the repo {{ globals.repo }} + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - task: + must_complete: true + repeat_prompt: true + async: true + name: analyze_security_entry_points + agents: + - seclab_taskflows.personalities.web_application_security_expert + model: code_analysis + user_prompt: | + The component is in the repo {{ result.repo }} and it has component id {{ result.app_id }} and location {{ result.location }}. + The notes of the component is as follows. + + {{ result.notes }} + + Fetch the security entry points component, then the user actions of this component. + Based on the security entry points, components, user actions and README.md and if available, SECURITY.md in the {{ globals.repo }}, + can you tell me what type of application this repo is and what kind of security boundary it has. + + For each security entry point, identify the type of the security entry point and its functionality. + These entry points should be performing some kind of security sensitive operations, such as authentication, authorization, input validation, sanitization, escaping etc. + Identify the functionality of the entry point and what kind of issue may arise if the implementation is flawed. + For example, an authentication entry point may be susceptible to authentication bypass, + an input validation entry point may be susceptible to injection issues if the input is not properly sanitized, + an authorization entry point may be susceptible to IDOR or access control bypass if there are no proper access control checks. + This task is only to identify the potential risks and issues that may arise from the type of the security entry point, + rather than auditing the implementation of the entry point, so do not be too specific about the issue, and do not + audit the implementation, just identify the potential risks based on the type and functionality of the security entry point. + However, you should still take care not to include issues that are of low severity or require + unrealistic attack scenarios such as misconfiguration or an already compromised system, or issues that requires + attacker control of trusted inputs such as whitelist or blacklist bypass that requires the attacker + to be able to modify the whitelist or blacklist. + + Reflect on your notes and check that the attack scenario meets the above requirements. Exclude low severity issues or + issues that require unrealistic circumstances such as misconfiguration or stolen credentials etc. + It is ok to conclude that the component is not likely to have security issues. + + If you think the issues satisfy the criteria, store a component issue entry for each type of issue identified. + + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.local_file_viewer + - seclab_taskflow_agent.toolboxes.memcache \ No newline at end of file diff --git a/src/seclab_taskflows/taskflows/audit/gather_security_entry_point_info.yaml b/src/seclab_taskflows/taskflows/audit/gather_security_entry_point_info.yaml new file mode 100644 index 0000000..68aa4cd --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit/gather_security_entry_point_info.yaml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" +model_config: seclab_taskflows.configs.model_config + +globals: + repo: +# Taskflow to analyze the existing information +taskflow: + - task: + model: code_analysis + must_complete: true + headless: true + max_steps: 100 + agents: + - seclab_taskflows.personalities.web_application_security_expert + user_prompt: | + For the repo {{ globals.repo }} iterate over all entry points and check if they are + functions that are used for security purposes, + such as authentication, authorization, input validation, sanitization, escaping etc. + Identify potential inputs that these functions are likely to check or sanitize. Store these as + the "type" of the security entry point, and the potential untrusted input that it is processing. + Some functions + may have inputs that are related to configurations, such as a blacklist of disallowed inputs, or a list of allowed users or roles. + These inputs are typically supplied by developers and should not be treated as untrusted inputs. + For this check the noted `file` and its line using the file viewer. + For each entry point identified as a security endpoint create a new security entry point using + the information gathered. + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - seclab_taskflows.toolboxes.gh_file_viewer + \ No newline at end of file diff --git a/src/seclab_taskflows/taskflows/audit/identify_applications.yaml b/src/seclab_taskflows/taskflows/audit/identify_applications.yaml index 6728157..ab60a1d 100644 --- a/src/seclab_taskflows/taskflows/audit/identify_applications.yaml +++ b/src/seclab_taskflows/taskflows/audit/identify_applications.yaml @@ -24,6 +24,7 @@ taskflow: - task: model: code_analysis must_complete: true + max_steps: 100 name: identify components description: Identify different components and purpose of the repo agents: @@ -37,11 +38,16 @@ taskflow: Apart from source code, it is also useful to look at the README.md and documentations in the repo. + Note that even a software library may contain application components. + For example, a web application framework may contain a default web application as an example or demo, + or it may contain an admin UI, which are meant to be used for production. + + Separate the components of the library based on their functionality. For example, full stack web framework may + have components for handling http requests, components for database access, components for rendering UI, etc. + For each application identified, store a new component entry in the database. Provide some brief notes about the functionality of the component with the entry. - If the repo is a library, store the root directory as a library component in the database and provide some notes about - the purpose and the usage of the library. Identify example, demo or test code in the repo. Skip analyzing these code and do not create a component entry for these. toolboxes: @@ -75,36 +81,12 @@ taskflow: {{ result.notes }} Analyze the entry points of this application. First check whether this component is a library or - an application. From previous analysis stated that: - - is_application ? {{ result.is_app }} - is_library ? {{ result.is_library }} - - ## Application - - For application components, you should consider the following: - - Does this application consume untrusted input, for example, web requests? Or does it interact with - other applications and can take input from them? User supplied arguments, - configuration files, web requests from hardcoded URLs or environment variables are not considered untrusted - input. - - However, if user arguments are pointing to some untrusted resources, then they can be considered as - untrusted input. For example, a parser may take a user argument that points to a file, and in this case, - the file is still considered untrusted even though it is supplied via user argument. - - ## Library - - For library code, identify the functions that are exposed to a library user. Consider the library user - as trusted and instead focus on functionalities that consume untrusted input when they are used for - building applications. - - Ignore applications within the library such as default UI, etc. as these are going - to be analyzed separately. Focus on the library code. - - For example, a web application framework may expose functionalities such as http request handlers and middleware, which - are typically used for handling http requests when used for building web applications. In this case, the - http requests handled by these handlers and middleware are considered untrusted. + an application. + {% if result.is_library %} + {% include 'seclab_taskflows.prompts.audit.entry_point_library' %} + {% else %} + {% include 'seclab_taskflows.prompts.audit.entry_point_application' %} + {% endif %} Only identify the untrusted inputs and entry points, do not analyze the security risks of the component or the entry point. @@ -170,39 +152,13 @@ taskflow: Do not consider security risk or potential bypasses or any potential privilege escalations that is outside of the intended use. Focus only on the intended use here. - - First check whether this component is a library or - an application. From previous analysis stated that: - - is_application ? {{ result.is_app }} - is_library ? {{ result.is_library }} - ## Application + {% if result.is_library %} + {% include 'seclab_taskflows.prompts.audit.user_action_library' %} + {% else %} + {% include 'seclab_taskflows.prompts.audit.user_action_application' %} + {% endif %} - For application components, you should consider the following: - - What actions can a user perform using the application? Can they exectue arbitrary code? Can they - make arbitrary web request? Can they read any local file? Consider restrictions that are placed on the actions - that a user can perform using the application, and what actions the user should not be allowed to do - when using the application. - - You should also state clearly the actions and privileges that a user can have when using the application. - - ## Library - - For library component, do the following instead: - When considering user privileges, consider the library user as trusted and focus on the - functionalities that are exposed to untrusted users when the library is used. For example, a web - application library may provide functionalities for the library user to run arbitrary script to configure - the application. These are not considered "allowed actions" from an untrusted user point of view. - - However, if a library exposes a REST api that allows arbitrary files to be read by unauthenticated users, - then it reading arbitrary files is consider an "allowed" activity. Note any "allow" activity and how - they are exposed to untrusted users, referencing the file and function in detail. - - Ignore applications within the library such as default UI, etc. as these are going - to be analyzed separately. Focus on the library code. - Store an user_action entry stating precising the file and line number where the allowed action is performed, and provide notes about the action. diff --git a/src/seclab_taskflows/taskflows/audit/identify_library_components.yaml b/src/seclab_taskflows/taskflows/audit/identify_library_components.yaml new file mode 100644 index 0000000..038df0c --- /dev/null +++ b/src/seclab_taskflows/taskflows/audit/identify_library_components.yaml @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +seclab-taskflow-agent: + filetype: taskflow + version: "1.0" +model_config: seclab_taskflows.configs.model_config + +globals: + repo: +# Taskflow to analyze the general contextual information of a project and classify the different applications within it +taskflow: + - task: + must_complete: true + headless: true + model: general_tasks + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Clear the memory cache and clear the repo context results for repo {{ globals.repo }} + toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + - seclab_taskflows.toolboxes.repo_context + - task: + model: code_analysis + must_complete: true + name: identify components + description: Identify different components and purpose of the repo + agents: + - seclab_taskflows.personalities.web_application_security_expert + user_prompt: | + The repo {{ globals.repo }} is a software library. Determine its functionality and its likely + use case. For example, is it a library for building web applications, or is it a library for + processing and transforming data, framework for running unit tests, etc. + + Use the source code, documentation and README.md to determine the likely use case of the library. + + Separate the components of the library based on their functionality. For example, full stack web framework may + have components for handling http requests, components for database access, components for rendering UI, etc. + + Note that even a software library may contain application components. + For example, a web application framework may contain a default web application as an example or demo, + or it may contain an admin UI, which are meant to be used for production. + + Identify example, demo or test code in the repo. Use these to further your understanding of how the + library is meant to be used and the potential risks of the library, but do not analyze these components + and do not create a component entry for these. + toolboxes: + - seclab_taskflows.toolboxes.gh_file_viewer + - seclab_taskflows.toolboxes.repo_context + + - task: + must_complete: true + exclude_from_context: true + model: general_tasks + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Fetch the components from the repo {{ globals.repo }}. + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - task: + model: code_analysis + must_complete: false + repeat_prompt: true + async: true + max_steps: 100 + name: entry point analysis + description: Identify entry points that allow untrusted users to interact with the component. + agents: + - seclab_taskflows.personalities.web_application_security_expert + user_prompt: | + The component is in {{ result.repo }} in the directory {{ result.location }}. These are the + notes for this component: + + {{ result.notes }} + + Analyze the entry points of this application. First check whether this component is a library or + an application. From previous analysis stated that: + + is_application ? {{ result.is_app }} + is_library ? {{ result.is_library }} + + ## Application + + For application components, you should consider the following: + + Does this application consume untrusted input, for example, web requests? Or does it interact with + other applications and can take input from them? User supplied arguments, + configuration files, web requests from hardcoded URLs or environment variables are not considered untrusted + input. + + However, if user arguments are pointing to some untrusted resources, then they can be considered as + untrusted input. For example, a parser may take a user argument that points to a file, and in this case, + the file is still considered untrusted even though it is supplied via user argument. + + ## Library + + For library code, identify the functions that are exposed to a library user. + Use the component notes, documentation, source code and examples to + identify these functions and determine how they are meant to be used and whether + they are meant to consume untrusted input. + + Consider the library user + as trusted and instead focus on functionalities that consume untrusted input when they are used for + building applications. + + For example, a web application framework may expose functionalities such as http request handlers and middleware, which + are typically used for handling http requests when used for building web applications. In this case, the + http requests handled by these handlers and middleware are considered untrusted. + + Sometimes it may not be clear whether a function is consuming untrusted input. In this case, + use hints from documentation, comments and usage examples to determine whether the + function is meant to consume untrusted input. + For example, a function may perform string formatting or processing URL, but the function + name and documentation strongly suggests that it is typically used for processing input + that comes from web requests. In this case, you can consider the function as consuming untrusted input. + + For library code, there may be functions that are not directly taking untrusted input, but are used as security + checks or sanitization. Make use of documentation, code comments, function names and usage examples to identify + these functions, and consider them to be entry points as well. + + Only identify the untrusted inputs and entry points, do not analyze the security risks of the component or the + entry point. + + Once you're finished with this, store an "entry_point" for each entry point identified, providing the precise + file and line number of this entry point, as well as the user inputs that it exposes. Use the "notes" field to + explain how the entry point is exposed to the end user and why the identified user inputs are untrusted. + Note that line number has to be integer. If an entry point spans multiple lines, then use the first line as the line + number. + + ## IMPORTANT: General Guidance that ALWAYS applies + + 1. Do NOT ask the user for permission to perform next steps, continue your + analysis autonomously until it is complete. + + 2. Reflect on your analysis for accuracy before returning it to the user. + We are only interested in results that you can clearly explain and + motivate as potentially vulnerable based on code examples. + + 3. Do NOT speculate. All answers should be motivated by code examples and + match the conditions of our described vulnerability pattern. Call out + any situation where you are unsure about your results. + + toolboxes: + - seclab_taskflows.toolboxes.gh_file_viewer + - seclab_taskflows.toolboxes.repo_context + - task: + must_complete: true + exclude_from_context: true + model: general_tasks + agents: + - seclab_taskflow_agent.personalities.assistant + user_prompt: | + Fetch the components from the repo {{ globals.repo }}. + toolboxes: + - seclab_taskflows.toolboxes.repo_context + - task: + model: code_analysis + must_complete: false + repeat_prompt: true + async: true + max_steps: 100 + name: user actions analysis + description: Identify actions that untrusted users are allowed performed the component. + agents: + - seclab_taskflows.personalities.web_application_security_expert + user_prompt: | + The component is in {{ result.repo }} in the directory {{ result.location }}. These are the + notes for this component: + + {{ result.notes }} + + Analyze the actions that an untrusted user is allowed to perform. The purpose of this task is not + about the security of the source code, but rather get an understanding of the purpose of the component and + what an untrusted user should be allowed to do under the intention of the component. + For example, the component may have an api that allows an untrusted user to run aribitrary command. In this + case, the intention and purpose of the component is for user to run arbitrary command, and so the user action in + this case is arbitrary code execution. + + Similarly, an api may allow a user to read and write files to a specific directory. In this case the intention and + the user action is to read and write to restricted directory. This is different to allowing a user read and write + to any directories. Be precise in these details when looking at user action. + + Do not consider security risk or potential bypasses or any potential privilege escalations that is outside of the + intended use. Focus only on the intended use here. + + First check whether this component is a library or + an application. From previous analysis stated that: + + is_application ? {{ result.is_app }} + is_library ? {{ result.is_library }} + + ## Application + + For application components, you should consider the following: + + What actions can a user perform using the application? Can they exectue arbitrary code? Can they + make arbitrary web request? Can they read any local file? Consider restrictions that are placed on the actions + that a user can perform using the application, and what actions the user should not be allowed to do + when using the application. + + You should also state clearly the actions and privileges that a user can have when using the application. + + ## Library + + For library component, do the following instead: + When considering user privileges, consider the library user as trusted and focus on the + functionalities that are exposed to untrusted users when the library is used. For example, a web + application library may provide functionalities for the library user to run arbitrary script to configure + the application. These are not considered "allowed actions" from an untrusted user point of view. + + However, if a library exposes a REST api that allows arbitrary files to be read by unauthenticated users, + then it reading arbitrary files is consider an "allowed" activity. Note any "allow" activity and how + they are exposed to untrusted users, referencing the file and function in detail. + + Ignore applications within the library such as default UI, etc. as these are going + to be analyzed separately. Focus on the library code. + + Store an user_action entry stating precising the file and line number where the allowed action is performed, and + provide notes about the action. + + Note that line number has to be integer. If an user action spans multiple lines, then use the first line as the line + number. + + ## IMPORTANT: General Guidance that ALWAYS applies + + 1. Do NOT ask the user for permission to perform next steps, continue your + analysis autonomously until it is complete. + + 2. Reflect on your analysis for accuracy before returning it to the user. + We are only interested in results that you can clearly explain and + motivate as potentially vulnerable based on code examples. + + 3. Do NOT speculate. All answers should be motivated by code examples and + match the conditions of our described vulnerability pattern. Call out + any situation where you are unsure about your results. + + toolboxes: + - seclab_taskflows.toolboxes.gh_file_viewer + - seclab_taskflows.toolboxes.repo_context \ No newline at end of file From 85cd5571d3aa0e2a10797cc2eba056a84f48e4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20St=C3=B6ckli?= Date: Fri, 12 Jun 2026 14:17:39 +0000 Subject: [PATCH 2/2] fix: linter findings --- src/seclab_taskflows/mcp_servers/local_gh_resources.py | 4 ++-- src/seclab_taskflows/mcp_servers/repo_context.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/seclab_taskflows/mcp_servers/local_gh_resources.py b/src/seclab_taskflows/mcp_servers/local_gh_resources.py index 2d61d24..3450844 100644 --- a/src/seclab_taskflows/mcp_servers/local_gh_resources.py +++ b/src/seclab_taskflows/mcp_servers/local_gh_resources.py @@ -75,8 +75,8 @@ def _parse_content_disposition_filename(header: str | None) -> str | None: """Extract the filename from a Content-Disposition header value.""" if not header: return None - for part in header.split(";"): - part = part.strip() + for raw_part in header.split(";"): + part = raw_part.strip() if part.lower().startswith("filename="): filename = part.split("=", 1)[1].strip().strip('"') return filename diff --git a/src/seclab_taskflows/mcp_servers/repo_context.py b/src/seclab_taskflows/mcp_servers/repo_context.py index a5e8622..96a4a3f 100644 --- a/src/seclab_taskflows/mcp_servers/repo_context.py +++ b/src/seclab_taskflows/mcp_servers/repo_context.py @@ -33,8 +33,8 @@ def app_to_dict(result): "repo": result.repo.lower(), "location": result.location, "notes": result.notes, - "is_app": True if result.is_app else False, - "is_library": True if result.is_library else False, + "is_app": bool(result.is_app), + "is_library": bool(result.is_library), } @@ -365,8 +365,8 @@ def get_app_issues(self, repo, component_id): "issue_type": issue.issue_type, "issue_notes": issue.notes, "issue_id": issue.id, - "is_library": True if app.is_library else False, - "is_app": True if app.is_app else False, + "is_library": bool(app.is_library), + "is_app": bool(app.is_app), } for app, issue in issues ]