...
| Code Block | ||
|---|---|---|
| ||
import netCDF4 as nc import os import glob import xarray as xr import boto3 from botocore.session import Session from botocore.handlers import validate_bucket_name import tempfile def load_S3(): project_id = 'FILL ME IN' s3_file(bucketname, filename): bucketname = 'FILL ME IN' access_key = 'FILL ME IN' secret_access_key = 'FILL ME IN' endpoint = 'https://s3.waw3-1.cloudferro.com' bucketname = project_id + ':' + bucketname botocore_session = Session() botocore_session.unregister('before-parameter-build.s3', validate_bucket_name) boto3.setup_default_session(botocore_session = botocore_session) s3 = boto3.client('s3', endpoint_url=endpoint, aws_access_key_id = access_key, aws_secret_access_key = secret_access_key) files = [] #List the objects in our bucket responsetmp = s3tempfile.list_objects(Bucket=bucketnameNamedTemporaryFile() fortc item in response['Contents']: files.append(item["Key"]) return s3, files, bucketname def load_s3_file(filename): s3, files, bucketname = load_S3() tmp = tempfile.NamedTemporaryFile()= boto3.s3.transfer.TransferConfig(io_chunksize=2621440) with open(tmp.name, 'wb') as f: s3.download_fileobj(bucketname, filename, f, Config=tc) dataSet = xr.open_dataset(tmp.name, engine='netcdf4') return dataSet def main(): netCDFfile = load_s3_file("myfile.nc") main(load_s3_file("mybucket", "myfile.nc") |
And an alternative and shorter version:
| Code Block |
|---|
import smart_open
bucketpath=smart_f = f"s3://{access_key}:{secret_access_key}@{endpoint}@{bucketname}/{obj_name}"
smart_f = smart_open.open(bucketpath, 'rb')
import h5py
h=h5py.File(smart_f)
print(h.keys()) |
If you're interested in more, I recommend taking a look at this article, which gives you a more detailed view into boto3's functionality (although it does emphasize on Amazon Web Services specifically, you can take a look at the Python code involved):
...