How to mock a ZipFile object in S3?-CodePudding

I'd like to test uploading an archive to S3 bucket. This archive is unzipped so I'd like to test the extracted JSON files as well. Here is what I want to test:

def get_files_from_s3(bucket_name, s3_prefix):
    files = []
    s3_resource = boto3.resource("s3")
    bucket = s3_resource.Bucket(bucket_name)
    response = bucket.objects.filter(Prefix=s3_prefix)
    for obj in response:
        if obj.key.endswidth('.zip'):
            buffer = BytesIO(obj.get()["Body"].read())
            print("Before unzipping the file")
            z = ZipFile(buffer)
            for filename in z.namelist():
                print("After unzipping the file")
                if filename == "files/file1.json":
                    for dict in self.parse_json(z, filename):
                    # store some data from the json
            data.append(...)
    return data

def parse_json(obj, file):
    with obj.open(file, "r") as f:
        data = json.loads(f.read())
    return data["some_key"]

and here is what I've tried in unit testing after getting an answer to my previous question here:

from moto import mock_s3

@pytest.fixture(scope='function')
def aws_credentials():
    """Mocked AWS Credentials, to ensure we're not touching AWS directly"""
    os.environ['AWS_ACCESS_KEY_ID'] = 'testing'
    os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing'
    os.environ['AWS_SECURITY_TOKEN'] = 'testing'
    os.environ['AWS_SESSION_TOKEN'] = 'testing'


@mock_s3
def test_get_files_from_s3(self, aws_credentials):
    s3 = boto3.resource('s3')
    bucket = s3.Bucket(self.bucket_name)
    # Create the bucket first, as we're interacting with an empty mocked 'AWS account'
    bucket.create()

    # Create some example files that are representative of what the S3 bucket would look like in production
    client = boto3.client('s3')
    client.put_object(Bucket=self.bucket_name, Key=s3_prefix   "file.zip", Body=open('local_file.zip', 'r'))
    client.put_object(Bucket=self.bucket_name, Key=s3_prefix   "file.nonzip", Body="...")

    # Retrieve the files again using whatever logic
    files = module.get_files_from_s3(BUCKET_NAME, S3_PREFIX)
    self.assertEqual(['file.zip'], files)

So my concern here is how to mock a ZipFile object when passing to moto? because when I passed the Body argument with the actual zip file that I wanted to send to the S3 client, I found this error: E UnicodeDecodeError: 'utf-8' codec can't decode byte 0x9b in position 10: invalid start byte

Another concern is how to pass that object to another function like parse_json()?

Thanks

CodePudding user response：

The Body-parameter only accepts bytes, which means that the open-method needs to return the byte-content of a file using the rb argument:

client.put_object(Bucket=self.bucket_name, Key=s3_prefix   "file.zip", Body=open('local_file.zip', 'rb'))

Note that this is not a Zipfile/mocking/moto issue, but a boto3 limitation.
If the second argument is just r, the content will be returned as a string. When trying to upload that, AWS will fail with this error message: TypeError: Unicode-objects must be encoded before hashing