cools commited on
Commit
de396d7
·
1 Parent(s): b76ebc1

Delete AWSHandler.py

Browse files
Files changed (1) hide show
  1. AWSHandler.py +0 -47
AWSHandler.py DELETED
@@ -1,47 +0,0 @@
1
- import boto3
2
- import os
3
- import pandas as pd
4
- import json
5
-
6
- def upload_files(origin_path, destination_path, aws_access_key, aws_secret_key):
7
- session = boto3.Session(aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key)
8
- s3 = session.resource('s3')
9
- bucket = s3.Bucket('gideon-corpus')
10
-
11
- for subdir, dirs, files in os.walk(origin_path):
12
- for file in files:
13
- full_path = os.path.join(subdir, file)
14
- with open(full_path, 'rb') as data:
15
- bucket.put_object(Key=destination_path + origin_path.split('/')[-1] + '/' + full_path[len(origin_path) + 1:], Body=data)
16
-
17
-
18
- def retrieve_logs():
19
- s3 = boto3.client('s3')
20
- logs_response = s3.get_object(Bucket='gideon-corpus', Key='logs/logs.csv')
21
- logs_df = pd.read_csv(logs_response['Body'])
22
- return logs_df
23
-
24
- def retrieve_casedocs(case_num): # Note: this is how stuff is stored on AWS
25
- s3 = boto3.client('s3')
26
- opinions_response = s3.get_object(Bucket='gideon-corpus', Key='Cases/' + case_num + '/opinions.csv')
27
- opinions_df = pd.read_csv(opinions_response['Body'])
28
- metadata_response = s3.get_object(Bucket='gideon-corpus', Key='Cases/' + case_num + '/metadata.json')
29
- metadata = json.loads(metadata_response['Body'].read().decode('utf-8'))
30
- return opinions_df, metadata
31
-
32
-
33
- def retrieve_all_casedocs(prefix):
34
- s3 = boto3.client('s3')
35
- subdirectories = set()
36
- paginator = s3.get_paginator('list_objects_v2')
37
- for result in paginator.paginate(Bucket="gideon-corpus", Prefix=prefix, Delimiter='/'):
38
- if result.get('CommonPrefixes'):
39
- subdirectories.update(subdir.get('Prefix') for subdir in result.get('CommonPrefixes'))
40
- subdirectories = list(subdirectories)
41
- subs = [s.split('/')[1] for s in subdirectories]
42
-
43
- case_docs = []
44
- for s in subs:
45
- opinions_df, metadata = retrieve_casedocs(s)
46
- case_docs.append((opinions_df, metadata))
47
- return case_docs