Breaking News: Grepper is joining You.com. Read the official announcement!

upload data to s3 bucket python

Add Answer

MrMalfunction answered on November 24, 2021 Popularity 10/10 Helpfulness 4/10

answer upload data to s3 bucket python

related python upload file to s3

related upload file to s3 python

related upload folder to s3 bucket python

related how to upload folder in s3 bucket using python

upload data to s3 bucket python

Comment

import boto
import boto.s3
import sys
from boto.s3.key import Key

AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''

bucket_name = AWS_ACCESS_KEY_ID.lower() + '-dump'
conn = boto.connect_s3(AWS_ACCESS_KEY_ID,
        AWS_SECRET_ACCESS_KEY)


bucket = conn.create_bucket(bucket_name,
    location=boto.s3.connection.Location.DEFAULT)

testfile = "replace this with an actual filename"
print 'Uploading %s to Amazon S3 bucket %s' % \
   (testfile, bucket_name)

def percent_cb(complete, total):
    sys.stdout.write('.')
    sys.stdout.flush()


k = Key(bucket)
k.key = 'my test file'
k.set_contents_from_filename(testfile,
    cb=percent_cb, num_cb=10)

Popularity 10/10 Helpfulness 4/10 Language python

Source: stackoverflow.com

Tags: python s3-bucket upload

Link to this answer
Share Copy Link

Contributed on Nov 24 2021

MrMalfunction

0 Answers Avg Quality 2/10

Closely Related Answers

python upload file to s3

Comment

import boto3

s3 = boto3.resource('s3')
BUCKET = "test"

s3.Bucket(BUCKET).upload_file("your/local/file", "dump/file")

Popularity 9/10 Helpfulness 5/10 Language python

Source: stackoverflow.com

Tags: file

Link to this answer
Share Copy Link

Contributed on Sep 05 2022

Guisardo

0 Answers Avg Quality 2/10

upload file to s3 python

Comment

# easy way
import boto3

s3 = boto3.resource(
    service_name='s3',
    region_name='Your region',
    aws_access_key_id='Your id',
    aws_secret_access_key="Your Key")

bucket = s3.Bucket('your bucket name')

bucket.upload_file(Filename='temp.csv', Key='temp.csv')
# file path __________↑                 ↑ 
# (which you want to upload)            |
#                                       |______path (where you want to upload inside bucket)

Popularity 10/10 Helpfulness 3/10 Language python

Source: Grepper

Tags: file

Link to this answer
Share Copy Link

Contributed on Jun 11 2022

Darkstar

0 Answers Avg Quality 2/10

upload folder to s3 bucket python

Comment

def uploadDirectory(path,bucketname):
        for root,dirs,files in os.walk(path):
            for file in files:
                s3C.upload_file(os.path.join(root,file),bucketname,file)

Popularity 9/10 Helpfulness 2/10 Language python

Source: stackoverflow.com

Tags: python py

Link to this answer
Share Copy Link

Contributed on Apr 26 2022

Ahraf khatri

0 Answers Avg Quality 2/10

how to upload folder in s3 bucket using python

Comment

""" upload one directory from the current working directory to aws """
from pathlib import Path
import os
import glob
import boto3

def upload_dir(localDir, awsInitDir, bucketName, tag, prefix='/'):
    """
    from current working directory, upload a 'localDir' with all its subcontents (files and subdirectories...)
    to a aws bucket
    Parameters
    ----------
    localDir :   localDirectory to be uploaded, with respect to current working directory
    awsInitDir : prefix 'directory' in aws
    bucketName : bucket in aws
    tag :        tag to select files, like *png
                 NOTE: if you use tag it must be given like --tag '*txt', in some quotation marks... for argparse
    prefix :     to remove initial '/' from file names

    Returns
    -------
    None
    """
    s3 = boto3.resource('s3')
    cwd = str(Path.cwd())
    p = Path(os.path.join(Path.cwd(), localDir))
    mydirs = list(p.glob('**'))
    for mydir in mydirs:
        fileNames = glob.glob(os.path.join(mydir, tag))
        fileNames = [f for f in fileNames if not Path(f).is_dir()]
        rows = len(fileNames)
        for i, fileName in enumerate(fileNames):
            fileName = str(fileName).replace(cwd, '')
            if fileName.startswith(prefix):  # only modify the text if it starts with the prefix
                fileName = fileName.replace(prefix, "", 1) # remove one instance of prefix
            print(f"fileName {fileName}")

            awsPath = os.path.join(awsInitDir, str(fileName))
            s3.meta.client.upload_file(fileName, bucketName, awsPath)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--localDir", help="which dir to upload to aws")
    parser.add_argument("--bucketName", help="to which bucket to upload in aws")
    parser.add_argument("--awsInitDir", help="to which 'directory' in aws")
    parser.add_argument("--tag", help="some tag to select files, like *png", default='*')
    args = parser.parse_args()

    # cd whatever is above your dir, then run it
    # (below assuming this script is in ~/git/hu-libraries/netRoutines/uploadDir2Aws.py )
    # in the example below you have directory structure ~/Downloads/IO
    # you copy full directory of ~/Downloads/IO to aws bucket markus1 to 'directory' 2020/IO
    # NOTE: if you use tag it must be given like --tag '*txt', in some quotation marks...

    # cd ~/Downloads
    # python ~/git/hu-libraries/netRoutines/uploadDir2Aws.py --localDir IO --bucketName markus1 --awsInitDir 2020
    upload_dir(localDir=args.localDir, bucketName=args.bucketName,
               awsInitDir=args.awsInitDir, tag=args.tag)

Popularity 7/10 Helpfulness 2/10 Language python

Source: stackoverflow.com

Tags: python s3-bucket s

Link to this answer
Share Copy Link

Contributed on Dec 23 2022

Satyam Shishodiya

0 Answers Avg Quality 2/10

upload data to s3 bucket python

Contents

More Related Answers

upload data to s3 bucket python

Closely Related Answers

python upload file to s3

upload file to s3 python

upload folder to s3 bucket python

how to upload folder in s3 bucket using python

Grepper

Documentation

Social

Legal

Contact

Oops, You will need to install Grepper and log-in to perform this action.