...
If you're interested in streaming netCDF files directly from S3 buckets, give this example these two examples a look:
| Code Block | ||
|---|---|---|
| ||
import netCDF4 as nc
import xarray as xr
import boto3
import tempfile
def load_s3_file(bucketname, filename):
access_key = 'FILL ME IN'
secret_access_key = 'FILL ME IN'
endpoint = 'https://s3.waw3-1.cloudferro.com'
s3 = boto3.client('s3', endpoint_url=endpoint,
aws_access_key_id = access_key,
aws_secret_access_key = secret_access_key)
tmp = tempfile.NamedTemporaryFile()
tc = boto3.s3.transfer.TransferConfig(io_chunksize=2621440)
with open(tmp.name, 'wb') as f:
s3.download_fileobj(bucketname, filename, f, Config=tc)
dataSet = xr.open_dataset(tmp.name, engine='netcdf4')
return dataSet
load_s3_file("mybucket", "myfile.nc") |
...