diff --git a/CHANGELOG.md b/CHANGELOG.md index 96aed99..1390e1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,15 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> +----- +## [1.0.17](https://github.com/asfadmin/Discovery-SearchAPI-v3/compare/v1.0.15...v1.0.16) +### Changed +- Searches with `granule_list` can use other search filters +- bump asf-search to v12.2.2 + - Baseline bugfix + - S1D support + - OPERA-S1 file sizes in jsonlite2 output + ------ ## [1.0.16](https://github.com/asfadmin/Discovery-SearchAPI-v3/compare/v1.0.15...v1.0.16) ### Changed diff --git a/requirements.txt b/requirements.txt index 4527f0f..b20d230 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ ujson==5.7.0 uvicorn==0.21.1 watchfiles==0.19.0 -asf-search[asf-enumeration]==12.1.0 +asf-search[asf-enumeration]==12.2.2 python-json-logger==2.0.7 pyshp==2.1.3 diff --git a/src/SearchAPI/application/asf_opts.py b/src/SearchAPI/application/asf_opts.py index a19cc71..245ef3d 100644 --- a/src/SearchAPI/application/asf_opts.py +++ b/src/SearchAPI/application/asf_opts.py @@ -295,10 +295,11 @@ def get_asf_opts(params: dict) -> asf.ASFSearchOptions: "Unbound wildcard searches not supported with SearchAPI." "Specify `maxresults` or use the asf-search python module directly (try `output=python` to download the equivalent script)" ) - if len([param for param in params if param not in ["collections", "maxResults"]]) > 1: - raise ValueError( - 'Cannot use search keywords "granule_list/product_list" with other search params' - ) + if "product_list" in params: + if len([param for param in params if param not in ["collections", "maxResults"]]) > 1: + raise ValueError( + 'Cannot use search keyword "product_list" with other search params' + ) if (flight_direction := params.get("flightDirection")) is not None: if isinstance(flight_direction, str) and len(flight_direction): diff --git a/tests/yml_tests/test_URLs.yml b/tests/yml_tests/test_URLs.yml index a33ccf7..b2c0f33 100644 --- a/tests/yml_tests/test_URLs.yml +++ b/tests/yml_tests/test_URLs.yml @@ -458,6 +458,31 @@ tests: expected file: x-python expected code: 200 +- granule_list with platform valid: + granule_list: S1B_S6_GRDH_1SDV_20190911T214309_20190911T214338_017995_021E10_5CCB + platform: S1 + output: json + + expected file: json + expected code: 200 + +- granule_list with date valid: + granule_list: S1B_S6_GRDH_1SDV_20190911T214309_20190911T214338_017995_021E10_5CCB + start: "2019-01-01T00:00:00Z" + output: json + + expected file: json + expected code: 200 + +- granule_list with other keywords valid: + intersectsWith: polygon((-91.1083+41.7191,-83.9568+41.4233,-83.9916+43.9781,-91.7194+42.8556,-91.1083+41.7191)) + platform: Sentinel-1A, Sentinel-1B + granule_list: S1A_IW_GRDH_1SDV_20220201T000545_20220201T000612_041712_04F694_C8B2 + output: json + + expected file: json + expected code: 200 + #In CMR reporting script, we report groupid as null on most datasets. Some datasets updated to expect blank file & moved to partial pass yaml - groupid Sentinel: groupid: S1B_S1DV_0492_0497_017567_041 diff --git a/tests/yml_tests/test_URLs_expected_400.yml b/tests/yml_tests/test_URLs_expected_400.yml index 02a7540..7437ee8 100644 --- a/tests/yml_tests/test_URLs_expected_400.yml +++ b/tests/yml_tests/test_URLs_expected_400.yml @@ -202,30 +202,7 @@ tests: expected file: error json expected code: 400 -- granule_list with platform invalid: - granule_list: S1B_S6_GRDH_1SDV_20190911T214309_20190911T214338_017995_021E10_5CCB - platform: S1 - output: csv - - expected file: error json - expected code: 400 - -- granule_list with date invalid: - granule_list: S1B_S6_GRDH_1SDV_20190911T214309_20190911T214338_017995_021E10_5CCB - start: "2019-01-01T00:00:00Z" - output: csv - - expected file: error json - expected code: 400 -- granule_list with other keywords invalid: - intersectsWith: polygon((-91.1083+41.7191,-83.9568+41.4233,-83.9916+43.9781,-91.7194+42.8556,-91.1083+41.7191)) - platform: Sentinel-1A,Sentinel-1B - granule_list: S1A_IW_GRDH_1SDV_20220201T000545_20220201T000612_041712_04F694_C8B2 - output: json - - expected file: error json - expected code: 400 - granule_list with product_list invalid: granule_list: S1B_S6_GRDH_1SDV_20190911T214309_20190911T214338_017995_021E10_5CCB diff --git a/tests/yml_tests/test_url_manager.py b/tests/yml_tests/test_url_manager.py index 6ec81e6..6f3839c 100644 --- a/tests/yml_tests/test_url_manager.py +++ b/tests/yml_tests/test_url_manager.py @@ -1,9 +1,9 @@ import ast -import requests, urllib # For talking w/ API -import json, csv # File stuff -import re # Opening/Reading the file stuff -from io import StringIO # Opening/Reading the file stuff -from copy import deepcopy # For making duplicate dicts +import requests, urllib # For talking w/ API +import json, csv # File stuff +import re # Opening/Reading the file stuff +from io import StringIO # Opening/Reading the file stuff +from copy import deepcopy # For making duplicate dicts # from error_msg import error_msg # For timezone/timestamp verification: @@ -13,28 +13,31 @@ # from pytz import timezone from fastapi.testclient import TestClient + # from SearchAPI.CMR import Input as test_input from SearchAPI.application.asf_opts import string_to_obj_map import asf_search -class test_URL_Manager(): +class test_URL_Manager: def __init__(self, client: TestClient, **args): self.client = client self.error_msg = "Reason: {0}\n" test_info = args["test_info"] self.output_type = None - if test_info.get('output') is not None: - self.output_type = test_info['output'] + if test_info.get("output") is not None: + self.output_type = test_info["output"] test_vars = args["test_type_vars"] api_info = args["config"].getoption("--api") test_api = api_info["this_api"] - url_parts = [test_api, test_vars["endpoint"]+"?"] - full_url = '/'.join(s.strip('/') for s in url_parts) # If both/neither have '/' between them, this still joins them correctly + url_parts = [test_api, test_vars["endpoint"] + "?"] + full_url = "/".join( + s.strip("/") for s in url_parts + ) # If both/neither have '/' between them, this still joins them correctly # If the test itself stated the maturity, it's testing it: if "use_maturity" in test_info: @@ -51,7 +54,7 @@ def __init__(self, client: TestClient, **args): if use_cmr_maturity: keywords.append("maturity=" + test_vars["maturity"]) self.query = full_url + "&".join(keywords) - self.error_msg = "Reason: {0}\n - URL: '{1}'".format("{0}",self.query) + self.error_msg = "Reason: {0}\n - URL: '{1}'".format("{0}", self.query) # Figure out if you should print stuff: if "print" not in test_info: @@ -70,17 +73,14 @@ def __init__(self, client: TestClient, **args): if assert_used: self.runAssertTests(test_info, status_code, content_type, file_content) - - def getKeywords(self, test_info): # DONT add these to url. (Used for tester). Add ALL others to allow testing keywords that don't exist reserved_keywords = ["title", "print", "api", "skip_file_check", "maturity", "use_maturity"] - asserts_keywords = ["expected file","expected code"] - + asserts_keywords = ["expected file", "expected code"] - assert_used = 0 != len([k for k,_ in test_info.items() if k in asserts_keywords]) + assert_used = 0 != len([k for k, _ in test_info.items() if k in asserts_keywords]) keywords = [] - for key,val in test_info.items(): + for key, val in test_info.items(): # If it's reserved, move on: if key in reserved_keywords or key in asserts_keywords: continue @@ -89,9 +89,9 @@ def getKeywords(self, test_info): keywords.append(str(key)) # If you're testing multiple SAME params, add each key-val pair: elif isinstance(val, type([])): - keywords.append(str(key)+"="+",".join(val)) + keywords.append(str(key) + "=" + ",".join(val)) else: - keywords.append(str(key)+"="+str(val)) + keywords.append(str(key) + "=" + str(val)) return keywords, assert_used def runQuery(self): @@ -99,15 +99,23 @@ def countToDict(html): try: count = int(html.rstrip()) except ValueError: - assert False, self.error_msg.format("API returned html that was not a count. (Error page?) \nHTML Page (First 500 chars): \n{0}\n".format(file_content[:500])) + assert False, self.error_msg.format( + "API returned html that was not a count. (Error page?) \nHTML Page (First 500 chars): \n{0}\n".format( + file_content[:500] + ) + ) return {"count": count} def csvToDict(file_content): - file_content = csv.reader(StringIO(file_content), delimiter=',') + file_content = csv.reader(StringIO(file_content), delimiter=",") file_content = [a for a in file_content] # Rotate it counter-clockwise, so that row[0] == key of csv. (based on https://stackoverflow.com/questions/8421337/rotating-a-two-dimensional-array-in-python) rotated_content = list(map(type([]), zip(*file_content))) - assert len(rotated_content) > 1, self.error_msg.format("API did not return a CSV.\nReturned content: (First 500 chars): \n{0}\n".format(file_content[:500])) + assert len(rotated_content) > 1, self.error_msg.format( + "API did not return a CSV.\nReturned content: (First 500 chars): \n{0}\n".format( + file_content[:500] + ) + ) file_content = {} for column in rotated_content: file_content[column[0]] = column[1:] @@ -116,9 +124,13 @@ def csvToDict(file_content): def downloadToDict(bulk_download_file): # Grab everything in the self.files field of the download script: - files = re.search(r'self.files\s*=\s*\[.*?\]', bulk_download_file, re.DOTALL) + files = re.search(r"self.files\s*=\s*\[.*?\]", bulk_download_file, re.DOTALL) if files == None: - assert False, self.error_msg.format("Problem reading download script! \nReturned content: (First 500 chars): \n{0}\n".format(bulk_download_file[:500])) + assert False, self.error_msg.format( + "Problem reading download script! \nReturned content: (First 500 chars): \n{0}\n".format( + bulk_download_file[:500] + ) + ) # Parse out each file-names, and make each one a str in a list: files = re.findall('"(.*?)"', files.group(0)) # add the fields and return: @@ -132,18 +144,22 @@ def asfSearchToDict(asf_search_file): try: data = ast.parse(asf_search_file) except SyntaxError: - return ValueError('Failed to parse generated asf-search python script') - + return ValueError("Failed to parse generated asf-search python script") + outputs = {} - compiled_data = compile(data, filename='', mode='exec') + compiled_data = compile(data, filename="", mode="exec") exec(compiled_data, None, outputs) - geojson_data = outputs.get('results', outputs.get('stack', asf_search.ASFSearchResults([]))).geojson() + geojson_data = outputs.get( + "results", outputs.get("stack", asf_search.ASFSearchResults([])) + ).geojson() - geojson_query = self.query.replace('python', 'geojson') + geojson_query = self.query.replace("python", "geojson") geojson_api = json.loads(self.client.get(geojson_query).content.decode("utf-8")) - + script_geojson = str(geojson_data) - assert script_geojson == str(geojson_api), 'asf-search python file output differed from equivalent api geojson output' + assert script_geojson == str(geojson_api), ( + "asf-search python file output differed from equivalent api geojson output" + ) return script_geojson def jsonToDict(json_data): @@ -152,7 +168,7 @@ def jsonToDict(json_data): count = 0 for result in json_data: count += 1 - for key,val in result.items(): + for key, val in result.items(): # Break apart nested lists if needed, (alows to forloop val): val = [val] if not isinstance(val, type([])) else val if key in file_content: @@ -166,20 +182,26 @@ def jsonToDict(json_data): return file_content h = self.client.head(self.query) - content_header = h.headers.get('content-type') + content_header = h.headers.get("content-type") try: file_content = self.client.get(self.query).content.decode("utf-8") except requests.exceptions.ChunkedEncodingError: - assert False, self.error_msg.format("Server returned no info. Normally means it's overloaded.") + assert False, self.error_msg.format( + "Server returned no info. Normally means it's overloaded." + ) # text/csv; charset=utf-8 try: - content_type = content_header.split('/')[1] + content_type = content_header.split("/")[1] print(f" - HIT CONTENT TYPE BEFORE 1: {content_type}") except AttributeError: - assert False, self.error_msg.format("Header is not formatted as expected. Header: {0}.\nFile Content (First 500 char): \n{1}\n".format(content_header, file_content[:500])) + assert False, self.error_msg.format( + "Header is not formatted as expected. Header: {0}.\nFile Content (First 500 char): \n{1}\n".format( + content_header, file_content[:500] + ) + ) # Take out the "csv; charset=utf-8", without crahsing on things that don't have a charset print(f" - HIT CONTENT TYPE BEFORE 2: {content_type}") - content_type = content_type.split(';')[0] if ';' in content_type else content_type + content_type = content_type.split(";")[0] if ";" in content_type else content_type print(f" - HIT CONTENT TYPE AFTER: {content_type}") ## COUNT / HTML: if content_type == "html": @@ -196,7 +218,7 @@ def jsonToDict(json_data): content_type = "blank csv" ## DOWNLOAD / PLAIN elif content_type == "plain" or content_type == "x-python": - if self.output_type == 'download': + if self.output_type == "download": file_content = downloadToDict(file_content) if file_content["count"] == 0: content_type = "blank download" @@ -204,7 +226,7 @@ def jsonToDict(json_data): content_type = "x-python" else: file_content = asfSearchToDict(file_content) - content_type = 'geojson' + content_type = "geojson" # how many granules are in the script: ## GEOJSON @@ -218,7 +240,11 @@ def jsonToDict(json_data): try: file_content = json.loads(file_content) except json.decoder.JSONDecodeError: - self.error_msg.format("Test returned json header, but content failed to load.\nContent (First 500 char):\n{0}\n".format(file_content)) + self.error_msg.format( + "Test returned json header, but content failed to load.\nContent (First 500 char):\n{0}\n".format( + file_content + ) + ) ## ERROR if "error" in file_content: content_type = "error json" @@ -232,7 +258,11 @@ def jsonToDict(json_data): file_content = jsonToDict(file_content) ## JSON else: - assert isinstance(file_content, type([])), self.error_msg.format("Response did not contain a list of results.\nContent (First 500 char):\n{0}\n".format(file_content)) + assert isinstance(file_content, type([])), self.error_msg.format( + "Response did not contain a list of results.\nContent (First 500 char):\n{0}\n".format( + file_content + ) + ) json_data = file_content[0] if json_data == []: content_type = "blank json" @@ -243,24 +273,34 @@ def jsonToDict(json_data): ## KML elif content_type == "vnd.google-earth.kml+xml": content_type = "kml" - blank_kml = '\n\n\nASF Datapool Search Results\nSearch Performed: \n\n\n'.replace(" ", "") + blank_kml = '\n\n\nASF Datapool Search Results\nSearch Performed: \n\n\n'.replace( + " ", "" + ) if file_content.replace(" ", "") == blank_kml: content_type = "blank kml" ## METALINK elif content_type == "metalink+xml": content_type = "metalink" - blank_metalink = '\n\nAlaska Satellite Facilityhttp://www.asf.alaska.edu/\n\n\n'.replace(" ","") - if file_content.replace(" ","") == blank_metalink: + blank_metalink = '\n\nAlaska Satellite Facilityhttp://www.asf.alaska.edu/\n\n\n'.replace( + " ", "" + ) + if file_content.replace(" ", "") == blank_metalink: content_type = "blank metalink" return h.status_code, content_type, file_content def runAssertTests(self, test_info, status_code, content_type, file_content): if "expected code" in test_info: - assert test_info["expected code"] == status_code, self.error_msg.format("Status codes is different than expected.") + assert test_info["expected code"] == status_code, self.error_msg.format( + "Status codes is different than expected." + ) if "count" in file_content and "maxResults" in test_info: - assert test_info["maxResults"] >= file_content["count"], self.error_msg.format("API returned too many results.") + assert test_info["maxResults"] >= file_content["count"], self.error_msg.format( + "API returned too many results." + ) if "expected file" in test_info: - assert test_info["expected file"] == content_type, self.error_msg.format("Different file type returned than expected.") + assert test_info["expected file"] == content_type, self.error_msg.format( + "Different file type returned than expected." + ) # If the tester added the override, don't check its contents: if "skip_file_check" in test_info and test_info["skip_file_check"] == True: return @@ -288,27 +328,36 @@ def checkFileContainsExpected(key, test_info, file_dict): if isinstance(poss_list, type([])): expect_type = type(poss_list[0]) # "found_param" is always a string. Convert it to match - if found_param.startswith('[') and found_param.endswith(']'): + if found_param.startswith("[") and found_param.endswith("]"): found_param = ast.literal_eval(found_param) for param in found_param: if param >= poss_list[0] and param <= poss_list[1]: found_in_list = True break - elif expect_type(found_param) >= poss_list[0] and expect_type(found_param) <= poss_list[1]: + elif ( + expect_type(found_param) >= poss_list[0] + and expect_type(found_param) <= poss_list[1] + ): found_in_list = True break # This part gets hit for single numbers, and strings. (ie "Platform"): else: expect_type = type(poss_list) - if isinstance(found_param, str) and found_param.startswith('[') and found_param.endswith(']'): + if ( + isinstance(found_param, str) + and found_param.startswith("[") + and found_param.endswith("]") + ): # expect_type = type(poss_list) found_param = ast.literal_eval(found_param) for param in found_param: if param == poss_list: - # if expect_type(param) >= poss_list[0] and expect_type(param) <= poss_list[1]: + # if expect_type(param) >= poss_list[0] and expect_type(param) <= poss_list[1]: found_in_list = True break - elif isinstance(found_param, list) or isinstance(found_param, tuple): + elif isinstance(found_param, list) or isinstance( + found_param, tuple + ): for param in found_param: if param == poss_list: found_in_list = True @@ -324,37 +373,53 @@ def checkFileContainsExpected(key, test_info, file_dict): found_in_list = True break except: - if float(poss_list[0]) <= float(found_param) and float(found_param) <= float(poss_list[1]) + 1: + if ( + float(poss_list[0]) <= float(found_param) + and float(found_param) <= float(poss_list[1]) + 1 + ): found_in_list = True break # If inner for-loop found it, break out of this one too: if found_in_list == True: break - assert found_in_list, self.error_msg.format(key + " declared, but not found in file. File contents for key:\n{0}\n".format(file_dict[key])) + assert found_in_list, self.error_msg.format( + key + + " declared, but not found in file. File contents for key:\n{0}\n".format( + file_dict[key] + ) + ) def checkMinMax(key, test_info, file_dict): - if "min"+key in test_info and key in file_dict: + if "min" + key in test_info and key in file_dict: for value in file_dict[key]: - number_type = type(test_info["min"+key]) - assert number_type(value) >= test_info["min"+key], self.error_msg.format("Value found smaller than min key.") - if "max"+key in test_info and key in file_dict: + number_type = type(test_info["min" + key]) + assert number_type(value) >= test_info["min" + key], self.error_msg.format( + "Value found smaller than min key." + ) + if "max" + key in test_info and key in file_dict: for value in file_dict[key]: - number_type = type(test_info["max"+key]) - assert number_type(value) <= test_info["max"+key], self.error_msg.format("Value found greater than max key.") + number_type = type(test_info["max" + key]) + assert number_type(value) <= test_info["max" + key], self.error_msg.format( + "Value found greater than max key." + ) # FOR tz_orig: The timezone the string came from. # Alaska = "US/Alaska", UTC = "UTC", Blank = whatever timezone you're in now def convertTimezoneUTC(time, tz_orig=None): # Assume if tz_orig is overriden, it's a string of what timezone they want. Else, get whatever timezone you're in tz_orig = get_localzone() if tz_orig == None else ZoneInfo(tz_orig) - + # If it's a string, convert it to datetime and localize it. # Else it's already datetime, just localize to the timezone: if isinstance(time, type("")): # Strip down a string, so it can be used in the format: "%Y-%m-%dT%H:%M:%S" - time = time.split(".")[0] # take of any milliseconds. Normally sec.000000 - time = time[:-1] if time.endswith("Z") else time # take off the 'Z' if it's on the end - time = time[:-3] if time.endswith("UTC") else time # take off the 'UTC' if it's on the end + time = time.split(".")[0] # take of any milliseconds. Normally sec.000000 + time = ( + time[:-1] if time.endswith("Z") else time + ) # take off the 'Z' if it's on the end + time = ( + time[:-3] if time.endswith("UTC") else time + ) # take off the 'UTC' if it's on the end # Convert to a datetime object: time = datetime.strptime(time, "%Y-%m-%dT%H:%M:%S") @@ -371,16 +436,26 @@ def checkDate(later_date=None, earlier_date=None): earlier_date = convertTimezoneUTC(earlier_date) for theDate in later_date: theDate = convertTimezoneUTC(theDate, tz_orig="UTC") - assert theDate >= earlier_date, self.error_msg.format("File has too small of a date. File: {0}, earlier than test date: {1}.".format(theDate, earlier_date)) + assert theDate >= earlier_date, self.error_msg.format( + "File has too small of a date. File: {0}, earlier than test date: {1}.".format( + theDate, earlier_date + ) + ) elif isinstance(earlier_date, type([])): later_date = convertTimezoneUTC(later_date) for theDate in earlier_date: theDate = convertTimezoneUTC(theDate, tz_orig="UTC") - assert later_date >= theDate, self.error_msg.format("File has too large of a date. File: {0}, later than test date: {1}.".format(theDate, later_date)) - else: # Else they both are a single date. Not sure if this is needed, but... + assert later_date >= theDate, self.error_msg.format( + "File has too large of a date. File: {0}, later than test date: {1}.".format( + theDate, later_date + ) + ) + else: # Else they both are a single date. Not sure if this is needed, but... earlier_date = convertTimezoneUTC(earlier_date) later_date = convertTimezoneUTC(later_date) - assert later_date >= earlier_date, self.error_msg.format("Date: {0} is earlier than date {1}.".format(later_date, earlier_date)) + assert later_date >= earlier_date, self.error_msg.format( + "Date: {0} is earlier than date {1}.".format(later_date, earlier_date) + ) def checkSeason(file_start_dates, file_end_dates, season_list): def date_to_nth_day(date): @@ -391,11 +466,13 @@ def date_to_nth_day(date): if len(file_start_dates) == len(file_end_dates): file_dates = zip(file_start_dates, file_end_dates) else: - assert False, self.error_msg.format("Error running test! Not same number of start and end dates.") + assert False, self.error_msg.format( + "Error running test! Not same number of start and end dates." + ) # If it's [300,5], turn it into [[300,365],[1,5]]. Else make it [[x,y]] if season_list[0] > season_list[1]: - season_list = [ [season_list[0],365],[1,season_list[1]] ] + season_list = [[season_list[0], 365], [1, season_list[1]]] else: season_list = [season_list] @@ -407,14 +484,14 @@ def date_to_nth_day(date): year_diff = abs(start_season.year - end_season.year) # First check if the product's date takes up an entire year: if year_diff >= 2 or start_season.month <= end_season.month and year_diff >= 1: - days_ranges = [[1,365]] + days_ranges = [[1, 365]] else: # Convert start/end points to ints: start = date_to_nth_day(start_season) end = date_to_nth_day(end_season) # Check if both dates exist in the same calendar year: if year_diff == 0: - days_ranges.append([start,end]) + days_ranges.append([start, end]) # append both halfs of the range: else: days_ranges.append([start, 365]) @@ -425,12 +502,17 @@ def date_to_nth_day(date): for season in season_list: for the_range in days_ranges: # If either boundry in the file is in what you ask for in the season list, you pass: - if (season[0] <= the_range[0] <= season[1]) or (season[0] <= the_range[1] <= season[1]): + if (season[0] <= the_range[0] <= season[1]) or ( + season[0] <= the_range[1] <= season[1] + ): season_range_hit = True break - assert season_range_hit, self.error_msg.format("Seasons not found in file. file ranges: {0}. yml range: {1}.".format(days_ranges, season_list)) - + assert season_range_hit, self.error_msg.format( + "Seasons not found in file. file ranges: {0}. yml range: {1}.".format( + days_ranges, season_list + ) + ) checkFileContainsExpected("Platform", test_info, file_content) checkFileContainsExpected("absoluteOrbit", test_info, file_content) @@ -465,8 +547,6 @@ def date_to_nth_day(date): checkMinMax("insarstacksize", test_info, file_content) checkMinMax("faradayrotation", test_info, file_content) - - def parseTestValues(self, test_info): # Turn string values to lists: mutatable_dict = deepcopy(test_info) @@ -477,79 +557,121 @@ def parseTestValues(self, test_info): val = urllib.parse.unquote_plus(str(val)) if key.lower() == "absoluteorbit": del mutatable_dict[key] - mutatable_dict["absoluteOrbit"] = string_to_obj_map[asf_search.validators.parse_int_or_range_list](val) + mutatable_dict["absoluteOrbit"] = string_to_obj_map[ + asf_search.validators.parse_int_or_range_list + ](val) elif key.lower() == "platform": del mutatable_dict[key] - mutatable_dict["Platform"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["Platform"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() in ["frame", "asfframe"]: del mutatable_dict[key] - mutatable_dict["asfframe"] = string_to_obj_map[asf_search.validators.parse_int_or_range_list](val) + mutatable_dict["asfframe"] = string_to_obj_map[ + asf_search.validators.parse_int_or_range_list + ](val) elif key.lower() == "granule_list": del mutatable_dict[key] - mutatable_dict["granule_list"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["granule_list"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() == "groupid": del mutatable_dict[key] - mutatable_dict["groupid"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["groupid"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() == "flightdirection": del mutatable_dict[key] - mutatable_dict["flightdirection"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["flightdirection"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() == "offnadirangle": del mutatable_dict[key] - mutatable_dict["offnadirangle"] = string_to_obj_map[asf_search.validators.parse_float_or_range_list](val) + mutatable_dict["offnadirangle"] = string_to_obj_map[ + asf_search.validators.parse_float_or_range_list + ](val) elif key.lower() == "polarization": del mutatable_dict[key] - mutatable_dict["polarization"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["polarization"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() == "relativeorbit": del mutatable_dict[key] - mutatable_dict["relativeorbit"] = string_to_obj_map[asf_search.validators.parse_int_or_range_list](val) + mutatable_dict["relativeorbit"] = string_to_obj_map[ + asf_search.validators.parse_int_or_range_list + ](val) elif key.lower() == "collectionname": del mutatable_dict[key] - mutatable_dict["collectionname"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["collectionname"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() in ["beammode", "beamswath"]: del mutatable_dict[key] - mutatable_dict["beammode"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["beammode"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() == "processinglevel": del mutatable_dict[key] - mutatable_dict["processinglevel"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["processinglevel"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() == "flightline": del mutatable_dict[key] - mutatable_dict["flightline"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["flightline"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() == "lookdirection": del mutatable_dict[key] - mutatable_dict["lookdirection"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["lookdirection"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) elif key.lower() == "instrument": del mutatable_dict[key] - mutatable_dict["instrument"] = string_to_obj_map[asf_search.validators.parse_string_list](val) + mutatable_dict["instrument"] = string_to_obj_map[ + asf_search.validators.parse_string_list + ](val) # elif key.lower() == "processingdate": # del mutatable_dict[key] # mutatable_dict["processingdate"] = asf_search.validators.parse_date(val.replace("+", " ")) elif key.lower() == "start": del mutatable_dict[key] - mutatable_dict["start"] = asf_search.validators.parse_date(val.replace("+", " ")) + mutatable_dict["start"] = asf_search.validators.parse_date( + val.replace("+", " ") + ) elif key.lower() == "end": del mutatable_dict[key] mutatable_dict["end"] = asf_search.validators.parse_date(val.replace("+", " ")) elif key.lower() == "season": del mutatable_dict[key] - mutatable_dict["season"] = asf_search.validators.parse_int_list(string_to_obj_map[asf_search.validators.parse_int_list](val)) + mutatable_dict["season"] = asf_search.validators.parse_int_list( + string_to_obj_map[asf_search.validators.parse_int_list](val) + ) # MIN/MAX variants # min/max BaselinePerp elif key.lower()[3:] == "baselineperp": del mutatable_dict[key] # Save the min/max key, all lower - mutatable_dict[key.lower()[0:3]+"baselineperp"] = string_to_obj_map[asf_search.validators.parse_float_or_range_list](val) + mutatable_dict[key.lower()[0:3] + "baselineperp"] = string_to_obj_map[ + asf_search.validators.parse_float_or_range_list + ](val) # min/max Doppler: elif key.lower()[3:] == "doppler": del mutatable_dict[key] - mutatable_dict[key.lower()[0:3]+"doppler"] = asf_search.validators.parse_float(val) + mutatable_dict[key.lower()[0:3] + "doppler"] = ( + asf_search.validators.parse_float(val) + ) # min/max InsarStackSize: elif key.lower()[3:] == "insarstacksize": del mutatable_dict[key] - mutatable_dict[key.lower()[0:3]+"insarstacksize"] = asf_search.validators.parse_int(val) - #min/max FaradayRotation: + mutatable_dict[key.lower()[0:3] + "insarstacksize"] = ( + asf_search.validators.parse_int(val) + ) + # min/max FaradayRotation: elif key.lower()[3:] == "faradayrotation": del mutatable_dict[key] - mutatable_dict[key.lower()[0:3]+"faradayrotation"] = asf_search.validators.parse_float(val) + mutatable_dict[key.lower()[0:3] + "faradayrotation"] = ( + asf_search.validators.parse_float(val) + ) except ValueError as e: assert False, self.error_msg.format("ValueError: {0}.".format(str(e))) @@ -650,7 +772,6 @@ def renameKeysToStandard(self, json_dict): json_dict["endtime"] = json_dict.pop(key) return json_dict - # assumes values are in the form of {key: [value1,value2]} def renameValsToStandard(self, json_dict): if "Platform" in json_dict: @@ -691,6 +812,7 @@ def renameValsToStandard(self, json_dict): json_dict["Platform"].append("Sentinel-1A") json_dict["Platform"].append("Sentinel-1B") json_dict["Platform"].append("Sentinel-1C") + json_dict["Platform"].append("Sentinel-1D") json_dict["Platform"].append("Sentinel-1 Interferogram (BETA)") # Sentinel-1A elif platform in ["SENTINEL-1A", "SA"]: @@ -703,6 +825,9 @@ def renameValsToStandard(self, json_dict): elif platform in ["SENTINEL-1C", "SC"]: json_dict["Platform"][i] = "Sentinel-1C" json_dict["Platform"].append("Sentinel-1 Interferogram (BETA)") + elif platform in ["SENTINEL-1D", "SD"]: + json_dict["Platform"][i] = "Sentinel-1D" + json_dict["Platform"].append("Sentinel-1 Interferogram (BETA)") # Sir-C elif platform in ["SIR-C"]: del json_dict["Platform"][i] @@ -723,7 +848,7 @@ def renameValsToStandard(self, json_dict): # DESCENDING if flightdirection in ["D", "DESC", "DESCENDING"]: json_dict["flightdirection"][i] = "DESCENDING" - #ASCENDING + # ASCENDING elif flightdirection in ["A", "ASC", "ASCENDING"]: json_dict["flightdirection"][i] = "ASCENDING" if "lookdirection" in json_dict: @@ -731,10 +856,10 @@ def renameValsToStandard(self, json_dict): if lookdirection == None: continue lookdirection = lookdirection.upper() - #LEFT + # LEFT if lookdirection in ["L", "LEFT"]: json_dict["lookdirection"][i] = "LEFT" - #RIGHT + # RIGHT elif lookdirection in ["R", "RIGHT"]: json_dict["lookdirection"][i] = "RIGHT" if "polarization" in json_dict: @@ -772,4 +897,3 @@ def renameValsToStandard(self, json_dict): if beammode[0:2] == "ST": json_dict["beammode"][i] = "STD" return json_dict -