I am trying to remove special characters (like forward slash, $, etc.) from JSON key name (not from JSON value). I have written a Python script to convert single line JSON to break my each record (meaning each JSON payload should start from the new line). I want to include the logic of removing/renaming special character in this Python script. My idea is to basically use regex and ignore everything except character and number ("[^a-zA-Z0-9]") but I am not able to put this logic in the code as it needs to be only done for the key/field and not for the value.
Python:
import json
import boto3
import base64
import re
output = []
def lambda_handler(event, context):
for record in event['records']:
payload = base64.b64decode(record['data']).decode('utf-8')
print('payload:', payload)
row_w_newline = payload "\n"
print('row_w_newline type:', type(row_w_newline))
row_w_newline = base64.b64encode(row_w_newline.encode('utf-8'))
output_record = {
'recordId': record['recordId'],
'result': 'Ok',
'data': row_w_newline
}
output.append(output_record)
print('Processed {} records.'.format(len(event['records'])))
return {'records': output}
Including sample JSON:
"ProductFields": {
"StandardsArn": "arn:aws:securityhub:::standards/aws-foundational-security-best-practices/v/1.0.0",
"RelatedAWSResources:0/name": "securityhub-ec2-instance-managed-by-ssm-dc0c9f18",
"RelatedAWSResources:0/type": "AWS::Config::ConfigRule",
"StandardsControlArn": "arn:aws:securityhub:us-east-1:492931852779:control/aws-foundational-security-best-practices/v/1.0.0/SSM.1",
"aws/securityhub/ProductName": "Security Hub",
"aws/securityhub/CompanyName": "AWS"
}
Something like this:
Input :
"RelatedAWSResources:0/name": "securityhub-ec2-instance-managed-by-ssm-dc0c9f18",
"RelatedAWSResources:0/type": "AWS::Config::ConfigRule",
"aws/securityhub/ProductName": "Security Hub",
"aws/securityhub/CompanyName": "AWS"
Output:
"RelatedAWSResources0name": "securityhub-ec2-instance-managed-by-ssm-dc0c9f18",
"RelatedAWSResources0type": "AWS::Config::ConfigRule",
"awssecurityhubProductName": "Security Hub",
"awssecurityhubCompanyName": "AWS"
Complete JSON:
{
"version": "0",
"id": "e517d9a2-84ed-a2fb-fce7-bf5ff02b271b",
"detail-type": "Security Hub Findings - Imported",
"source": "aws.securityhub",
"account": "220307202362",
"time": "2021-10-19T07:26:54Z",
"region": "us-west-2",
"resources": [
"arn:aws:securityhub:us-west-2::product/aws/securityhub/arn:aws:securityhub:us-west-2:220307202362:subscription/cis-aws-foundations-benchmark/v/1.2.0/1.1/finding/c5e2448c-a877-428e-adfe-4f21ab0b2d23"
],
"detail": {
"findings": [
{
"ProductArn": "arn:aws:securityhub:us-west-2::product/aws/securityhub",
"Types": [
"Software and Configuration Checks/Industry and Regulatory Standards/CIS AWS Foundations Benchmark"
],
"Description": "The \"root\" account has unrestricted access to all resources in the AWS account. It is highly recommended that the use of this account be avoided.",
"Compliance": {
"Status": "FAILED",
"StatusReasons": [
{
"Description": "Multi-region CloudTrail with the required configuration does not exist in the account.",
"ReasonCode": "CLOUDTRAIL_MULTI_REGION_NOT_PRESENT"
}
]
},
"ProductName": "Security Hub",
"FirstObservedAt": "2021-09-27T19:34:26.016Z",
"CreatedAt": "2021-09-27T19:34:26.016Z",
"LastObservedAt": "2021-10-19T07:26:50.588Z",
"CompanyName": "AWS",
"FindingProviderFields": {
"Types": [
"Software and Configuration Checks/Industry and Regulatory Standards/CIS AWS Foundations Benchmark"
],
"Severity": {
"Normalized": 30,
"Label": "LOW",
"Product": 30,
"Original": "LOW"
}
},
"ProductFields": {
"StandardsGuideArn": "arn:aws:securityhub:::ruleset/cis-aws-foundations-benchmark/v/1.2.0",
"StandardsGuideSubscriptionArn": "arn:aws:securityhub:us-west-2:220307202362:subscription/cis-aws-foundations-benchmark/v/1.2.0",
"RuleId": "1.1",
"RecommendationUrl": "https://docs.aws.amazon.com/console/securityhub/standards-cis-1.1/remediation",
"StandardsControlArn": "arn:aws:securityhub:us-west-2:220307202362:control/cis-aws-foundations-benchmark/v/1.2.0/1.1",
"aws/securityhub/ProductName": "Security Hub",
"aws/securityhub/CompanyName": "AWS",
"aws/securityhub/annotation": "Multi region CloudTrail with the required configuration does not exist in the account",
"Resources:0/Id": "arn:aws:iam::220307202362:root",
"aws/securityhub/FindingId": "arn:aws:securityhub:us-west-2::product/aws/securityhub/arn:aws:securityhub:us-west-2:220307202362:subscription/cis-aws-foundations-benchmark/v/1.2.0/1.1/finding/c5e2448c-a877-428e-adfe-4f21ab0b2d23"
},
"Remediation": {
"Recommendation": {
"Text": "For directions on how to fix this issue, consult the AWS Security Hub CIS documentation.",
"Url": "https://docs.aws.amazon.com/console/securityhub/standards-cis-1.1/remediation"
}
},
"SchemaVersion": "2018-10-08",
"GeneratorId": "arn:aws:securityhub:::ruleset/cis-aws-foundations-benchmark/v/1.2.0/rule/1.1",
"RecordState": "ACTIVE",
"Title": "1.1 Avoid the use of the \"root\" account",
"Workflow": {
"Status": "NEW"
},
"Severity": {
"Normalized": 30,
"Label": "LOW",
"Product": 30,
"Original": "LOW"
},
"UpdatedAt": "2021-10-19T07:26:47.032Z",
"WorkflowState": "NEW",
"AwsAccountId": "220307202362",
"Region": "us-west-2",
"Id": "arn:aws:securityhub:us-west-2:220307202362:subscription/cis-aws-foundations-benchmark/v/1.2.0/1.1/finding/c5e2448c-a877-428e-adfe-4f21ab0b2d23",
"Resources": [
{
"Partition": "aws",
"Type": "AwsAccount",
"Region": "us-west-2",
"Id": "AWS::::Account:220307202362"
}
]
}
]
}
}
CodePudding user response：
import json
import base64
json_string= '{"Relate:0/name": "securityhub-ec2-instance-managed-by-ssm-dc0c9f18","RelatedAWSResources:0/type": "AWS::Config::ConfigRule","aws/securityhub/ProductName": "Security Hub","aws/securityhub/CompanyName": "AWS"}'
json_dict = json.loads(json_string)
print(json_dict.keys())
new_dict = {}
for key in json_dict.keys():
new_dict[base64.b64encode(key.encode('utf-8'))]= json_dict[key]
print(new_dict)
here you can see i have converted keys to base64, in that same place you could call a function to replace special characters.
CodePudding user response：
I'd suggest creating a helper function that strips all punctuation from a string. I'm actually using something similar in a project of mine.
import string
from typing import Optional, Iterable, Union
delete_dict = {sp_character: '' for sp_character in string.punctuation}
PUNCT_TABLE = str.maketrans(delete_dict)
def strip_punctuation(s: str,
exclude_chars: Optional[Union[str, Iterable]] = None) -> str:
"""
Remove punctuation and spaces from a string.
If `exclude_chars` is passed, certain characters will not be removed
from the string.
"""
punct_table = PUNCT_TABLE.copy()
if exclude_chars:
for char in exclude_chars:
punct_table.pop(ord(char), None)
# Next, remove the desired punctuation from the string
return s.translate(punct_table)
Usage:
cleaned_keys = {'RelatedAWSResources:0/name': 'RelatedAWSResources0name',
'RelatedAWSResources:0/@!type#$%': 'RelatedAWSResources0type',
'aws/securityhub/ProductName': 'awssecurityhubProductName',
'aws/securityhub/CompanyName': 'awssecurityhubCompanyName'}
for key, expected_key in cleaned_keys.items():
actual_key = strip_punctuation(key)
assert actual_key == expected_key, f'Unexpected result for key: {key}'