NEANIAS Gitlab

Commit 6f253690 authored by Carlos H. Brandt's avatar Carlos H. Brandt
Browse files

move download/ode script to top, add bounding-box query

parent e67b967f
......@@ -18,3 +18,6 @@ Note:
* So far, the packages are not versionized neither is the container.
* TODO: bind notebooks to specific version of container and/or packages (environment).
## ToC
* [ODE REST API](ode_rest_api.md)
from ode import request_product
from ode import find_product_file
def download_file_silent(url, filename):
"""
Download file (silently)
Usage:
download_file_silent('http://web4host.net/5MB.zip', 'local_filename.zip')
"""
import requests
r = requests.get(url)
with open(filename,'wb') as f:
f.write(r.content)
return
def download_file_progress(url, filename, verbose=False):
"""
Download file with progressbar
Usage:
download_file_progress('http://web4host.net/5MB.zip', 'local_filename.zip')
"""
import requests
import tqdm
r = requests.get(url, stream=True)
file_size = int(r.headers['Content-Length'])
chunk = 1
chunk_size=1024
num_bars = int(file_size / chunk_size)
if verbose:
print(dict(file_size=file_size))
print(dict(num_bars=num_bars))
with open(filename, 'wb') as fp:
for chunk in tqdm.tqdm(
r.iter_content(chunk_size=chunk_size)
, total= num_bars
, unit = 'KB'
, desc = filename
, leave = True # progressbar stays
):
fp.write(chunk)
return
def download_file(url, filename=None, progress_on=False):
import os
if not filename:
local_filename = os.path.join('.', url.split('/')[-1])
else:
local_filename = filename
print('--> Downloading file {} ..'.format(local_filename))
if progress_on:
download_file_progress(url, local_filename)
else:
download_file_silent(url, local_filename)
print('--> File downloaded.'.format(local_filename))
return local_filename
def download_files(urls, descriptors, filenames=None, path=None, progress_on=False):
import os
assert isinstance(urls, (list, tuple))
assert isinstance(descriptors, (list, tuple))
assert len(urls) == len(descriptors)
if not filenames:
path = path.strip() if path != None and len(path.strip()) > 0 else '.'
filenames = [os.path.join(path, url.split('/')[-1]) for url in urls]
assert len(filenames) == len(urls), "List of 'filenames' must match length of 'urls'"
files_downloaded = {}
for url, descriptor, filename in zip(urls,descriptors, filenames):
try:
_fn = download_file(url, filename, progress_on)
files_downloaded[descriptor] = _fn
except:
pass
return files_downloaded
def get_product(product_id, file_types, descriptors, api_endpoint, path=None, progress_on=True):
import shutil
import sys
import os
pid = product_id
base_path = path or '.'
if not os.path.isdir(base_path):
os.mkdir(base_path)
print('Querying ODE for Product {} ..'.format(pid))
req = request_product(pid, api_endpoint)
if not req.status_code == 200:
print('ODE query for product {} failed.'.format(pid), file=sys.stderr)
return False
available_files = req.json()['ODEResults']['Products']['Product']['Product_files']['Product_file']
files_selected = [find_product_file(available_files, _ft, descriptors) for _ft in file_types]
try:
path = os.path.join(base_path, pid)
if not os.path.isdir(path):
os.mkdir(path)
print('-> Downloading Image data and Browse products for {} ..'.format(pid))
urls=[_fs['URL'] for _fs in files_selected]
products_downloaded = download_files(urls=urls, descriptors=file_types, path=path, progress_on=progress_on)
print('-> Data products downloaded.')
except:
print("-X ERROR while downloading '{pid}' products. Cleaning out path '{pid}'".format(pid=pid))
shutil.rmtree(pid)
print('Done.')
return products_downloaded
def get_products(product_ids, file_types, descriptors, api_endpoint, path=None, progress_on=True):
'''
List of Product-IDs
'''
base_path = path or '.'
products_data = {}
pids = product_ids
for i, pid in enumerate(product_ids):
print('Get-Products: ({i}/{size}) {pid}'.format(i=i, size=len(pids), pid=pid))
try:
products_downloaded = get_product(pid,
file_types=file_types,
descriptors=descriptors,
progress_on=progress_on,
api_endpoint=api_endpoint,
path=base_path)
products_data[pid] = products_downloaded
except:
pass
print('---')
return products_data
\ No newline at end of file
API_URL = 'https://oderest.rsl.wustl.edu/live2'
DESCRIPTORS = {
'ctx': {
'product_image': ('Description','PRODUCT DATA FILE WITH LABEL'),
'browse_image': ('Description','BROWSE IMAGE'),
'browse_thumbnail': ('Description','THUMBNAIL IMAGE')
},
'hirise': {
'product_image': ('Description', 'PRODUCT DATA FILE'),
'product_label': ('Description', 'PRODUCT LABEL FILE'),
'browse_image': ('Description', 'BROWSE'),
'browse_thumbnail': ('Description', 'THUMBNAIL')
}
}
DATASETS = {
'ctx': {
'ptype': 'edr',
'instr': 'ctx',
'host': 'mro'
},
'hirise': {
'ptype': 'rdrv11',
'instr': 'hirise',
'host': 'mex'
}
}
def request_product(PRODUCTID, api_endpoint):
import requests
payload = dict(
query='product',
results='fmp',
output='JSON',
productid=PRODUCTID
)
#payload.update({'pretty':True})
return requests.get(api_endpoint, params=payload)
def request_products(api_endpoint, bbox, target=None, host=None, instr=None, ptype=None):
"""
bbox = {
'minlat': [-65:65],
'minlat': [-65:65],
'westlon': [0:360],
'eastlon': [0:360]
}
'ptype' (eg, "rdrv11") is used only when 'instr' is also defined (e.g, "hirise").
"""
import requests
payload = dict(
query='product',
results='fmpc',
output='JSON',
loc='f',
minlat=bbox['minlat'],
maxlat=bbox['maxlat'],
westlon=bbox['westlon'],
eastlon=bbox['eastlon']
)
if target:
payload.update({'target':target})
if host:
payload.update({'ihid':host})
if instr:
payload.update({'iid':instr})
if ptype:
payload.update({'pt':ptype})
#payload.update({'pretty':True})
return requests.get(api_endpoint, params=payload)
def requested_product_files(request):
product_files = request.json()['ODEResults']['Products']['Product']['Product_files']['Product_file']
return product_files
def readout_product_files(product_json):
product_files = product_json['Product_files']['Product_file']
return product_files
def readout_product_footprint(product_json):
# 'Footprint_geometry' and 'Footprint_C0_geometry' may contain 'GEOMETRYCOLLECTION'
# when the footprint cross the meridian in "c180" or "c0" frames
#product_geom = request.json()['ODEResults']['Products']['Product']['Footprint_geometry']
#product_geom = request.json()['ODEResults']['Products']['Product']['Footprint_C0_geometry']
product_geom = product_json['Footprint_GL_geometry']
return product_geom
#
#TODO: ler uma lista de produtos: foreach product in "['ODEResults']['Products']"
#
def readout_product_meta(product_json):
product = {}
# <pdsid>ESP_011712_1820_COLOR</pdsid>
product['id'] = product_json['pdsid']
# <ihid>MRO</ihid>
product['mission'] = product_json['ihid']
# <iid>HIRISE</iid>
product['inst'] = product_json['iid']
# <pt>RDRV11</pt>
product['type'] = product_json['pt']
return product
def requested_products(request):
assert request.status_code == 200 and request.json()['ODEResults']['Status'].lower() == 'success'
products = request.json()['ODEResults']['Products']['Product']
assert isinstance(products, list), "Was expecting 'list', got '{}' instead".format(type(products))
return products
def find_product_file(product_files, product_type, descriptors=DESCRIPTORS):
_key,_val = descriptors[product_type]
pfl = list(filter(lambda pf:pf[_key]==_val, product_files))
_multiple_matches = "I was expecting only one Product matching ptype '{}' bu got '{}'."
assert len(pfl) == 1, _multiple_matches.format(product_type, len(pfl))
return pfl[0]
This diff is collapsed.
This diff is collapsed.
from ode import request_product
from ode import find_product_file
def download_file_silent(url, filename):
"""
Download file (silently)
Usage:
download_file_silent('http://web4host.net/5MB.zip', 'local_filename.zip')
"""
import requests
r = requests.get(url)
with open(filename,'wb') as f:
f.write(r.content)
return
def download_file_progress(url, filename, verbose=False):
"""
Download file with progressbar
Usage:
download_file_progress('http://web4host.net/5MB.zip', 'local_filename.zip')
"""
import requests
import tqdm
r = requests.get(url, stream=True)
file_size = int(r.headers['Content-Length'])
chunk = 1
chunk_size=1024
num_bars = int(file_size / chunk_size)
if verbose:
print(dict(file_size=file_size))
print(dict(num_bars=num_bars))
with open(filename, 'wb') as fp:
for chunk in tqdm.tqdm(
r.iter_content(chunk_size=chunk_size)
, total= num_bars
, unit = 'KB'
, desc = filename
, leave = True # progressbar stays
):
fp.write(chunk)
return
def download_file(url, filename=None, progress_on=False):
import os
if not filename:
local_filename = os.path.join('.', url.split('/')[-1])
else:
local_filename = filename
print('--> Downloading file {} ..'.format(local_filename))
if progress_on:
download_file_progress(url, local_filename)
else:
download_file_silent(url, local_filename)
print('--> File downloaded.'.format(local_filename))
return local_filename
def download_files(urls, descriptors, filenames=None, path=None, progress_on=False):
import os
assert isinstance(urls, (list, tuple))
assert isinstance(descriptors, (list, tuple))
assert len(urls) == len(descriptors)
if not filenames:
path = path.strip() if path != None and len(path.strip()) > 0 else '.'
filenames = [os.path.join(path, url.split('/')[-1]) for url in urls]
assert len(filenames) == len(urls), "List of 'filenames' must match length of 'urls'"
files_downloaded = {}
for url, descriptor, filename in zip(urls,descriptors, filenames):
try:
_fn = download_file(url, filename, progress_on)
files_downloaded[descriptor] = _fn
except:
pass
return files_downloaded
def get_product(product_id, file_types, api_endpoint, path=None, progress_on=True):
import shutil
import sys
import os
pid = product_id
base_path = path or '.'
if not os.path.isdir(base_path):
os.mkdir(base_path)
print('Querying ODE for Product {} ..'.format(pid))
req = request_product(pid, api_endpoint)
if not req.status_code == 200:
print('ODE query for product {} failed.'.format(pid), file=sys.stderr)
return False
available_files = req.json()['ODEResults']['Products']['Product']['Product_files']['Product_file']
files_selected = [find_product_file(available_files, _ft) for _ft in file_types]
try:
path = os.path.join(base_path, pid)
if not os.path.isdir(path):
os.mkdir(path)
print('-> Downloading Image data and Browse products for {} ..'.format(pid))
urls=[_fs['URL'] for _fs in files_selected]
descriptors = file_types
products_downloaded = download_files(urls=urls, descriptors=descriptors, path=path, progress_on=progress_on)
print('-> Data products downloaded.')
except:
print("-X ERROR while downloading '{pid}' products. Cleaning out path '{pid}'".format(pid=pid))
shutil.rmtree(pid)
print('Done.')
return products_downloaded
def get_products(product_ids, file_types, api_endpoint, path=None, progress_on=True):
'''
List of Product-IDs
'''
base_path = path or '.'
products_data = {}
pids = product_ids
for i, pid in enumerate(product_ids):
print('Get-Products: ({i}/{size}) {pid}'.format(i=i, size=len(pids), pid=pid))
try:
products_downloaded = get_product(pid,
file_types=file_types,
progress_on=progress_on,
api_endpoint=api_endpoint,
path=base_path)
products_data[pid] = products_downloaded
except:
pass
print('---')
return products_data
\ No newline at end of file
../download.py
\ No newline at end of file
API_URL = 'https://oderest.rsl.wustl.edu/live2'
DESCRIPTORS = {
'product_image': ('Description','PRODUCT DATA FILE WITH LABEL'),
'browse_image': ('Description','BROWSE IMAGE'),
'browse_thumbnail': ('Description','THUMBNAIL IMAGE')
}
def request_product(PRODUCTID, api_endpoint):
import requests
payload = dict(
query='product',
results='fmp',
output='JSON',
productid=PRODUCTID
)
#payload.update({'pretty':True})
return requests.get(api_endpoint, params=payload)
def requested_product_files(request):
product_files = request.json()['ODEResults']['Products']['Product']['Product_files']['Product_file']
return product_files
def find_product_file(product_files, product_type, descriptors=DESCRIPTORS):
_key,_val = descriptors[product_type]
pfl = list(filter(lambda pf:pf[_key]==_val, product_files))
_multiple_matches = "I was expecting only one Product matching ptype '{}' bu got '{}'."
assert len(pfl) == 1, _multiple_matches.format(product_type, len(pfl))
return pfl[0]
../ode.py
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
"""
# ----------
# Listing:
bbox = {
'latmin': -1,
'latmax': 1,
'lonmin': 179,
'lonmax': 181
}
ode_results = find_intersect_products(boundingbox=bbox, dataset='mro/ctx/edr')
geojson = json2geojson(ode_results)
# ----------
# Download
path_archive_0 = '/mnt/das/mro/ctx/edr'
for product in geojson['FeaturesCollection']:
out = download_product(product, path=path_archive_0)
if out:
product.update({'path_image': out})
# ----------
# Pre-Processing (Calibration / Map-projection)
## calibration
lvl2_list = []
for product in geojson['FeaturesCollection']:
_lvl0 = product['path_image']
_bname = _lvl0[:-4]
_lvl1 = _bname + '.cub'
_lvl2 = _bname + '.cal.cub'
try:
mroctx2isis(FROM=_lvl0, TO=_lvl1)
spiceinit(FROM=_lvl1)
ctxcal(FROM=_lvl1, TO=_lvl2)
lvl2_list.append(_lvl2)
except:
continue
## map-project
map_defs = 'sinusoidal.map'
mosrange(FROMLIST=lvl2_list, TO=map_defs, projection='SINUSOIDAL', precision=0)
lvl3_list = []
for _lvl2 in lvl2_list:
_lvl3 = _lvl2.replace('.cub','.map.cub')
try:
cam2map(FROM=_lvl2, TO=_lvl3, map=map_defs, pixres=map)
lvl3_list.append(_lvl3)
except:
continue
# ----------
# Processing (Mosaic)
product = 'mosaic.cub'
automos(FROMLIST=lvl3_list, mosaic=product)
"""
def search(dataset_ID, bbox=None, product_IDs=None):
"""
Search for products in dataset
Search can access either (local) shapefiles or ODE's API
"""
use_local_shapefiles = False
if use_local_shapefiles:
df = query_shapefile(dataset_ID, bbox, product_IDs)
else:
# use_ode_restapi
df = query_ode(dataset_ID, bbox, product_IDs)
return df
def find(product_IDs, output):
* Listing job/processing:
- image_url = "query"
- header_url = "query"
- geometry = geometry
- UTCStart = UTCStart
- UTCEnd = UTCEnd
- productId = PRODUCT_ID
- datasetId = INSTRUMENT_HOST_ID + INSTRUMENT_ID + ODE_TYPE_ABBREVIATION
- productType = TYPE
```
def script ( product_IDs, output ):
def run (product_IDs):
return [products_URLs]
# Write geojson to 'output'
return True/False
```
* Download:
- image_path = "download"
- header_path = "download"
```
def script ( feature, path, output ):
def run (feature, path)
assert image_url != "" and header_url
# merge path + filename
# download to absolute-path
return absolute-path or None
filename = run(feature, path)
return # update feature
```
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.