|
1 | 1 | import os |
2 | 2 | import requests |
| 3 | +import boto3 |
| 4 | +from botocore.exceptions import NoCredentialsError |
3 | 5 | import sys |
4 | | -import subprocess |
5 | 6 | import tomli |
6 | | -import tools |
7 | 7 |
|
8 | | -import rasterio as rio |
9 | | -from rasterio.warp import calculate_default_transform, reproject, Resampling |
10 | 8 |
|
| 9 | +class MapboxUploader: |
11 | 10 |
|
12 | | -def on_mapbox(flight): |
13 | | - """Check if the mbtiles file has already been uploaded to mapbox""" |
| 11 | + def __init__(self, access_token, username): |
| 12 | + self.access_token = access_token |
| 13 | + self.username = username |
| 14 | + self.base_url = f"https://api.mapbox.com/uploads/v1/{self.username}" |
| 15 | + |
| 16 | + def request_s3_credentials(self): |
| 17 | + credentials_url = f"{self.base_url}/credentials?access_token={self.access_token}" |
| 18 | + response = requests.post(credentials_url) |
| 19 | + if response.status_code == 200: |
| 20 | + return response.json() |
| 21 | + else: |
| 22 | + raise Exception(f"Failed to retrieve S3 credentials. Status code: {response.status_code}") |
| 23 | + |
| 24 | + def upload_to_s3(self, file_path, s3_credentials): |
| 25 | + s3_client = boto3.client( |
| 26 | + 's3', |
| 27 | + aws_access_key_id=s3_credentials['accessKeyId'], |
| 28 | + aws_secret_access_key=s3_credentials['secretAccessKey'], |
| 29 | + aws_session_token=s3_credentials['sessionToken'], |
| 30 | + region_name='us-east-1' # Use the appropriate AWS region |
| 31 | + ) |
| 32 | + try: |
| 33 | + s3_client.upload_file(file_path, s3_credentials['bucket'], s3_credentials['key']) |
| 34 | + except NoCredentialsError: |
| 35 | + print("Credentials not available.") |
| 36 | + raise |
| 37 | + |
| 38 | + def create_upload(self, s3_credentials, tileset_id): |
| 39 | + upload_url = f"{self.base_url}?access_token={self.access_token}" |
| 40 | + headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache'} |
| 41 | + data = {"url": s3_credentials['url'], "tileset": f"{self.username}.{tileset_id}"} |
| 42 | + response = requests.post(upload_url, json=data, headers=headers) |
| 43 | + if response.status_code == 201: |
| 44 | + return response.json() |
| 45 | + else: |
| 46 | + raise Exception(f"Failed to create upload. Status code: {response.status_code}") |
| 47 | + |
| 48 | + def retrieve_upload_status(self, upload_id): |
| 49 | + status_url = f"{self.base_url}/{upload_id}?access_token={self.access_token}" |
| 50 | + response = requests.get(status_url) |
| 51 | + if response.status_code in {200, 201}: |
| 52 | + return response.json() |
| 53 | + else: |
| 54 | + raise Exception(f"Failed to retrieve upload status. Status code: {response.status_code}") |
| 55 | + |
| 56 | + |
| 57 | +def get_credentials(): |
| 58 | + """Get credentials from mapbox.ini""" |
14 | 59 | with open("/blue/ewhite/everglades/mapbox/mapbox.ini", "rb") as f: |
15 | 60 | toml_dict = tomli.load(f) |
16 | | - token = toml_dict['mapbox']['access-token'] |
17 | | - api_base_url = "https://api.mapbox.com/v4" |
18 | | - tileset_id = f"bweinstein.{flight}" |
19 | | - url = f"{api_base_url}/{tileset_id}.json?access_token={token}" |
20 | | - response = requests.get(url) |
21 | | - return response.status_code == 200 |
22 | | - |
23 | | - |
24 | | -def upload(path, year, site, force_upload=False): |
25 | | - # Create output filename |
26 | | - basename = os.path.splitext(os.path.basename(path))[0] |
27 | | - flight = basename.replace("_projected", "") |
28 | | - if tools.get_event(basename) == "primary": |
29 | | - # If from the primary flight strip any extra metadata from filename |
30 | | - flight = "_".join(basename.split('_')[0:4]) |
31 | | - mbtiles_dir = os.path.join("/blue/ewhite/everglades/mapbox/", year, site) |
32 | | - if not os.path.exists(mbtiles_dir): |
33 | | - os.makedirs(mbtiles_dir) |
34 | | - mbtiles_filename = os.path.join(mbtiles_dir, f"{flight}.mbtiles") |
35 | | - |
36 | | - # Generate tiles |
37 | | - print("Creating mbtiles file") |
38 | | - subprocess.run([ |
39 | | - "rio", "mbtiles", path, "-o", mbtiles_filename, "--zoom-levels", "17..24", "-j", "4", "-f", "PNG", |
40 | | - "--progress-bar" |
41 | | - ]) |
42 | | - |
43 | | - # Upload to mapbox |
44 | | - print("Uploading to mapbox") |
45 | | - if force_upload or not on_mapbox(flight): |
46 | | - subprocess.run(["mapbox", "upload", f"bweinstein.{flight}", mbtiles_filename]) |
47 | | - else: |
48 | | - print(f"{flight} is already on Mapbox, not uploading. To force reupload use --force-upload") |
49 | | - |
50 | | - return mbtiles_filename |
| 61 | + access_token = toml_dict['mapbox']['access-token'] |
| 62 | + return access_token |
51 | 63 |
|
52 | 64 |
|
53 | 65 | if __name__ == "__main__": |
54 | | - path = sys.argv[1] |
55 | | - split_path = os.path.normpath(path).split(os.path.sep) |
56 | | - year = split_path[6] |
57 | | - site = split_path[7] |
58 | | - if len(sys.argv) == 3 and sys.argv[2] == "--force-upload": |
59 | | - force_upload = True |
60 | | - else: |
61 | | - force_upload = False |
62 | | - upload(path, year, site, force_upload=force_upload) |
| 66 | + file_path = sys.argv[1] |
| 67 | + access_token = get_credentials() |
| 68 | + username = 'bweinstein' |
| 69 | + uploader = MapboxUploader(access_token, username) |
| 70 | + # Step 1: Request S3 credentials |
| 71 | + s3_credentials = uploader.request_s3_credentials() |
| 72 | + |
| 73 | + # Step 2: Upload to Mapbox's S3 staging bucket |
| 74 | + uploader.upload_to_s3(file_path, s3_credentials) |
| 75 | + filename_without_extension = os.path.splitext(os.path.basename(file_path))[0] |
| 76 | + tileset_id = filename_without_extension |
| 77 | + |
| 78 | + # Step 3: Create an upload to matbox |
| 79 | + upload_data = uploader.create_upload(s3_credentials, tileset_id) |
| 80 | + upload_id = upload_data['id'] |
| 81 | + upload_status = uploader.retrieve_upload_status(upload_id) |
| 82 | + print(f"Upload status: {upload_status}") |
0 commit comments