From 4ef9c4d5a85b394c01b05b0d2b530d21b808b598 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 16 Sep 2024 02:03:11 +0530 Subject: [PATCH 1/4] Add "OTHER" as supported hackerone asset --- web/reNgine/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/reNgine/definitions.py b/web/reNgine/definitions.py index abe599156..b264c52a6 100644 --- a/web/reNgine/definitions.py +++ b/web/reNgine/definitions.py @@ -565,4 +565,4 @@ ) # Bountyhub Definitions -HACKERONE_ALLOWED_ASSET_TYPES = ["WILDCARD", "DOMAIN", "IP_ADDRESS", "URL"] \ No newline at end of file +HACKERONE_ALLOWED_ASSET_TYPES = ["WILDCARD", "DOMAIN", "IP_ADDRESS", "URL", "OTHER"] \ No newline at end of file From fd9d11bc4971d7a88825d06b820ae1dbc35202fe Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 16 Sep 2024 02:03:33 +0530 Subject: [PATCH 2/4] add other in frontend --- web/static/custom/bountyhub.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/static/custom/bountyhub.js b/web/static/custom/bountyhub.js index 1dbcf30a6..87a97907d 100644 --- a/web/static/custom/bountyhub.js +++ b/web/static/custom/bountyhub.js @@ -457,7 +457,7 @@ function populateAssetAccordion(data) { const hackerone_handle = data.attributes.handle; const program_name = data.attributes.name; const assetTypes = { - WILDCARD: [], DOMAIN: [], IP_ADDRESS: [], CIDR: [], URL: [] + WILDCARD: [], DOMAIN: [], IP_ADDRESS: [], CIDR: [], URL: [], OTHER: [] }; data.relationships.structured_scopes.data.forEach(scope => { From 4e07099b7580d8b443702aa589b3beba13f6bfd4 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 16 Sep 2024 02:04:03 +0530 Subject: [PATCH 3/4] Add util function to validate the hackerone asset type using reges, support for domain, ip, url and wildcard --- web/api/views.py | 13 +++++++++---- web/reNgine/common_func.py | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/web/api/views.py b/web/api/views.py index fcea8abd9..0ee9a0fb3 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -25,7 +25,7 @@ from reNgine.celery import app from reNgine.common_func import * from reNgine.database_utils import * -from reNgine.definitions import ABORTED_TASK +from reNgine.definitions import ABORTED_TASK, HACKERONE_ALLOWED_ASSET_TYPES from reNgine.tasks import * from reNgine.llm import * from reNgine.utilities import is_safe_path @@ -64,8 +64,6 @@ class HackerOneProgramViewSet(viewsets.ViewSet): API_BASE = 'https://api.hackerone.com/v1/hackers' - ALLOWED_ASSET_TYPES = ["WILDCARD", "DOMAIN", "IP_ADDRESS", "CIDR", "URL"] - def list(self, request): try: sort_by = request.query_params.get('sort_by', 'age') @@ -179,9 +177,16 @@ def program_details(self, request, pk=None): if program_details: filtered_scopes = [ scope for scope in program_details.get('relationships', {}).get('structured_scopes', {}).get('data', []) - if scope.get('attributes', {}).get('asset_type') in self.ALLOWED_ASSET_TYPES + if scope.get('attributes', {}).get('asset_type') in HACKERONE_ALLOWED_ASSET_TYPES ] + # refine filtered_scopes with that are valid assets + filtered_scopes = [ + scope for scope in filtered_scopes + if is_valid_asset_identifier(scope['attributes']['asset_identifier']) + ] + + program_details['relationships']['structured_scopes']['data'] = filtered_scopes return Response(program_details) diff --git a/web/reNgine/common_func.py b/web/reNgine/common_func.py index ad58a94a8..9f8c460be 100644 --- a/web/reNgine/common_func.py +++ b/web/reNgine/common_func.py @@ -1646,3 +1646,27 @@ def get_ips_from_cidr_range(target): return [str(ip) for ip in ipaddress.IPv4Network(target, False)] except Exception as e: logger.error(f'{target} is not a valid CIDR range. Skipping.') + + +def is_valid_asset_identifier(identifier): + """ + This function will check if the asset is supported by reNgine + As of now supported assets are: + - Domain + - IP Address + - URL + - Wildcard Domain for example *.example.com, *.example.co.uk etc + Args: + identifier: str: Identifier to check + """ + domain_regex = r'^(?!-)[A-Za-z0-9-]{1,63}(? Date: Mon, 16 Sep 2024 02:17:02 +0530 Subject: [PATCH 4/4] verify wildcards, remove them during import --- web/api/shared_api_tasks.py | 10 ++++++---- web/reNgine/common_func.py | 18 +++++++++++++++++- web/reNgine/database_utils.py | 3 +++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/web/api/shared_api_tasks.py b/web/api/shared_api_tasks.py index d21ca23fa..472421616 100644 --- a/web/api/shared_api_tasks.py +++ b/web/api/shared_api_tasks.py @@ -1,7 +1,7 @@ # include all the celery tasks to be used in the API, do not put in tasks.py import requests -from reNgine.common_func import create_inappnotification, get_hackerone_key_username +from reNgine.common_func import create_inappnotification, get_hackerone_key_username, is_valid_asset_identifier from reNgine.definitions import PROJECT_LEVEL_NOTIFICATION, HACKERONE_ALLOWED_ASSET_TYPES from reNgine.celery import app from reNgine.database_utils import bulk_import_targets @@ -62,12 +62,14 @@ def fetch_program_details_from_hackerone(program_handle): # in future release we will add this in target out_of_scope # we need to filter the scope that are supported by reNgine now - if asset_type in HACKERONE_ALLOWED_ASSET_TYPES and eligible_for_submission: + if asset_type in HACKERONE_ALLOWED_ASSET_TYPES \ + and eligible_for_submission \ + and is_valid_asset_identifier(asset_identifier): assets.append(asset_identifier) # in some cases asset_type is OTHER and may contain the asset - elif asset_type == 'OTHER' and ('.' in asset_identifier or asset_identifier.startswith('http')): - assets.append(asset_identifier) + # elif asset_type == 'OTHER' and ('.' in asset_identifier or asset_identifier.startswith('http')): + # assets.append(asset_identifier) # cleanup assets assets = list(set(assets)) diff --git a/web/reNgine/common_func.py b/web/reNgine/common_func.py index 9f8c460be..632cc3e36 100644 --- a/web/reNgine/common_func.py +++ b/web/reNgine/common_func.py @@ -1658,6 +1658,8 @@ def is_valid_asset_identifier(identifier): - Wildcard Domain for example *.example.com, *.example.co.uk etc Args: identifier: str: Identifier to check + Returns: + bool: True if identifier is valid, False otherwise """ domain_regex = r'^(?!-)[A-Za-z0-9-]{1,63}(?