new script for fetching the download lists

amazon-tile_id.csv

+T17LQJ
+T17LQK
+T17LQL
+T17LRG
+T17LRH
+T17LRJ
+T17LRK
+T17LRL
+T17MPM
+T17MPN
+T17MPP
+T17MPQ
+T17MPR
+T17MPS
+T17MPT
+T17MPU
+T17MPV
+T17MQM
+T17MQN
+T17MQP
+T17MQQ
+T17MQR
+T17MQS
+T17MQT
+T17MQU
+T17MQV
+T17MRM
+T17MRN
+T17MRP
+T17MRQ
+T17MRR
+T17MRS
+T17MRT
+T17MRU
+T17MRV
+T17NPA
+T17NPB
+T17NPC
+T17NPD
+T17NQA
+T17NQB
+T17NQC
+T17NQD
+T17NQE
+T17NQF
+T17NQG
+T17NRA
+T17NRB
+T17NRC
+T17NRD
+T17NRE
+T17NRF
+T17NRG
+T18LTM
+T18LTN
+T18LTP
+T18LTQ
+T18LTR
+T18LUJ
+T18LUK
+T18LUL
+T18LUM
+T18LUN
+T18LUP
+T18LUQ
+T18LUR
+T18LVH
+T18LVJ
+T18LVK
+T18LVL
+T18LVM
+T18LVN
+T18LVP
+T18LVQ
+T18LVR
+T18LWH
+T18LWJ
+T18LWK
+T18LWL
+T18LWM
+T18LWN
+T18LWP
+T18LWQ
+T18LWR
+T18LXH
+T18LXJ
+T18LXK
+T18LXL
+T18LXM
+T18LXN
+T18LXP
+T18LXQ
+T18LXR
+T18LYH
+T18LYJ
+T18LYK
+T18LYL
+T18LYM
+T18LYN
+T18LYP
+T18LYQ
+T18LYR
+T18LZH
+T18LZJ
+T18LZK
+T18LZL
+T18LZM
+T18LZN
+T18LZP
+T18LZQ
+T18LZR
+T18MTA
+T18MTB
+T18MTC
+T18MTD
+T18MTE
+T18MTS
+T18MTT
+T18MTU
+T18MTV
+T18MUA
+T18MUB
+T18MUC
+T18MUD
+T18MUE
+T18MUS
+T18MUT
+T18MUU
+T18MUV
+T18MVA
+T18MVB
+T18MVC
+T18MVD
+T18MVE
+T18MVS
+T18MVT
+T18MVU
+T18MVV
+T18MWA
+T18MWB
+T18MWC
+T18MWD
+T18MWE
+T18MWS
+T18MWT
+T18MWU
+T18MWV
+T18MXA
+T18MXB
+T18MXC
+T18MXD
+T18MXE
+T18MXS
+T18MXT
+T18MXU
+T18MXV
+T18MYA
+T18MYB
+T18MYC
+T18MYD
+T18MYE
+T18MYS
+T18MYT
+T18MYU
+T18MYV
+T18MZA
+T18MZB
+T18MZC
+T18MZD
+T18MZE
+T18MZS
+T18MZT
+T18MZU
+T18MZV
+T18NTF
+T18NTG
+T18NTH
+T18NTJ
+T18NTK
+T18NTL
+T18NTM
+T18NUF
+T18NUG
+T18NUH
+T18NUJ
+T18NUK
+T18NUL
+T18NUM
+T18NVF
+T18NVG
+T18NVH
+T18NVJ
+T18NVK
+T18NVL
+T18NVM
+T18NWF
+T18NWG
+T18NWH
+T18NWJ
+T18NWK
+T18NWL
+T18NWM
+T18NXF
+T18NXG
+T18NXH
+T18NXJ
+T18NXK
+T18NXL
+T18NXM
+T18NYF
+T18NYG
+T18NYH
+T18NYJ
+T18NYK
+T18NYL
+T18NYM
+T18NZF
+T18NZG
+T18NZH
+T18NZJ
+T18NZK
+T18NZL
+T18NZM
+T19LBC
+T19LBD
+T19LBE
+T19LBF
+T19LBG
+T19LBH
+T19LBJ
+T19LBK
+T19LBL
+T19LCC
+T19LCD
+T19LCE
+T19LCF
+T19LCG
+T19LCH
+T19LCJ
+T19LCK
+T19LCL
+T19LDC
+T19LDD
+T19LDE
+T19LDF
+T19LDG
+T19LDH
+T19LDJ
+T19LDK
+T19LDL
+T19LEC
+T19LED
+T19LEE
+T19LEF
+T19LEG
+T19LEH
+T19LEJ
+T19LEK
+T19LEL
+T19LFC
+T19LFD
+T19LFE
+T19LFF
+T19LFG
+T19LFH
+T19LFJ
+T19LFK
+T19LFL
+T19LGC
+T19LGD
+T19LGE
+T19LGF
+T19LGG
+T19LGH
+T19LGJ
+T19LGK
+T19LGL
+T19LHC
+T19LHD
+T19LHE
+T19LHF
+T19LHG
+T19LHH
+T19LHJ
+T19LHK
+T19LHL
+T19MBM
+T19MBN
+T19MBP
+T19MBQ
+T19MBR
+T19MBS
+T19MBT
+T19MBU
+T19MBV
+T19MCM
+T19MCN
+T19MCP
+T19MCQ
+T19MCR
+T19MCS
+T19MCT
+T19MCU
+T19MCV
+T19MDM
+T19MDN
+T19MDP
+T19MDQ
+T19MDR
+T19MDS
+T19MDT
+T19MDU
+T19MDV
+T19MEM
+T19MEN
+T19MEP
+T19MEQ
+T19MER
+T19MES
+T19MET
+T19MEU
+T19MEV
+T19MFM
+T19MFN
+T19MFP
+T19MFQ
+T19MFR
+T19MFS
+T19MFT
+T19MFU
+T19MFV
+T19MGM
+T19MGN
+T19MGP
+T19MGQ
+T19MGR
+T19MGS
+T19MGT
+T19MGU
+T19MGV
+T19MHM
+T19MHN
+T19MHP
+T19MHQ
+T19MHR
+T19MHS
+T19MHT
+T19MHU
+T19MHV
+T19NBA
+T19NBB
+T19NBC
+T19NBD
+T19NBE
+T19NBF
+T19NBG
+T19NCA
+T19NCB
+T19NCC
+T19NCD
+T19NCE
+T19NCF
+T19NCG
+T19NDA
+T19NDB
+T19NDC
+T19NDD
+T19NDE
+T19NDF
+T19NDG
+T19NEA
+T19NEB
+T19NEC
+T19NED
+T19NEE
+T19NEF
+T19NEG
+T19NFA
+T19NFB
+T19NFC
+T19NFD
+T19NFE
+T19NFF
+T19NFG
+T19NGA
+T19NGB
+T19NGC
+T19NGD
+T19NGE
+T19NGF
+T19NGG
+T19NHA
+T19NHB
+T19NHC
+T19NHD
+T19NHE
+T19NHF
+T19NHG
+T20LKH
+T20LKJ
+T20LKK
+T20LKL
+T20LKM
+T20LKN
+T20LKP
+T20LKQ
+T20LKR
+T20LLH
+T20LLJ
+T20LLK
+T20LLL
+T20LLM
+T20LLN
+T20LLP
+T20LLQ
+T20LLR
+T20LMH
+T20LMJ
+T20LMK
+T20LML
+T20LMM
+T20LMN
+T20LMP
+T20LMQ
+T20LMR
+T20LNH
+T20LNJ
+T20LNK
+T20LNL
+T20LNM
+T20LNN
+T20LNP
+T20LNQ
+T20LNR
+T20LPH
+T20LPJ
+T20LPK
+T20LPL
+T20LPM
+T20LPN
+T20LPP
+T20LPQ
+T20LPR
+T20LQH
+T20LQJ
+T20LQK
+T20LQL
+T20LQM
+T20LQN
+T20LQP
+T20LQQ
+T20LQR
+T20LRH
+T20LRJ
+T20LRK
+T20LRL
+T20LRM
+T20LRN
+T20LRP
+T20LRQ
+T20LRR
+T20MKA
+T20MKB
+T20MKC
+T20MKD
+T20MKE
+T20MKS
+T20MKT
+T20MKU
+T20MKV
+T20MLA
+T20MLB
+T20MLC
+T20MLD
+T20MLE
+T20MLS
+T20MLT
+T20MLU
+T20MLV
+T20MMA
+T20MMB
+T20MMC
+T20MMD
+T20MME
+T20MMS
+T20MMT
+T20MMU
+T20MMV
+T20MNA
+T20MNB
+T20MNC
+T20MND
+T20MNE
+T20MNS
+T20MNT
+T20MNU
+T20MNV
+T20MPA
+T20MPB
+T20MPC
+T20MPD
+T20MPE
+T20MPS
+T20MPT
+T20MPU
+T20MPV
+T20MQA
+T20MQB
+T20MQC
+T20MQD
+T20MQE
+T20MQS
+T20MQT
+T20MQU
+T20MQV
+T20MRA
+T20MRB
+T20MRC
+T20MRD
+T20MRE
+T20MRS
+T20MRT
+T20MRU
+T20MRV
+T20NKF
+T20NKG
+T20NKH
+T20NKJ
+T20NKK
+T20NKL
+T20NKM
+T20NLF
+T20NLG
+T20NLH
+T20NLJ
+T20NLK
+T20NLL
+T20NLM
+T20NMF
+T20NMG
+T20NMH
+T20NMJ
+T20NMK
+T20NML
+T20NMM
+T20NNF
+T20NNG
+T20NNH
+T20NNJ
+T20NNK
+T20NNL
+T20NNM
+T20NPF
+T20NPG
+T20NPH
+T20NPJ
+T20NPK
+T20NPL
+T20NPM
+T20NQF
+T20NQG
+T20NQH
+T20NQJ
+T20NQK
+T20NQL
+T20NQM
+T20NRF
+T20NRG
+T20NRH
+T20NRJ
+T20NRK
+T20NRL
+T20NRM
+T21LTC
+T21LTD
+T21LTE
+T21LTF
+T21LTG
+T21LTH
+T21LTJ
+T21LTK
+T21LTL
+T21LUC
+T21LUD
+T21LUE
+T21LUF
+T21LUG
+T21LUH
+T21LUJ
+T21LUK
+T21LUL
+T21LVC
+T21LVD
+T21LVE
+T21LVF
+T21LVG
+T21LVH
+T21LVJ
+T21LVK
+T21LVL
+T21LWC
+T21LWD
+T21LWE
+T21LWF
+T21LWG
+T21LWH
+T21LWJ
+T21LWK
+T21LWL
+T21LXC
+T21LXD
+T21LXE
+T21LXF
+T21LXG
+T21LXH
+T21LXJ
+T21LXK
+T21LXL
+T21LYC
+T21LYD
+T21LYE
+T21LYF
+T21LYG
+T21LYH
+T21LYJ
+T21LYK
+T21LYL
+T21LZC
+T21LZD
+T21LZE
+T21LZF
+T21LZG
+T21LZH
+T21LZJ
+T21LZK
+T21LZL
+T21MTM
+T21MTN
+T21MTP
+T21MTQ
+T21MTR
+T21MTS
+T21MTT
+T21MTU
+T21MTV
+T21MUM
+T21MUN
+T21MUP
+T21MUQ
+T21MUR
+T21MUS
+T21MUT
+T21MUU
+T21MUV
+T21MVM
+T21MVN
+T21MVP
+T21MVQ
+T21MVR
+T21MVS
+T21MVT
+T21MVU
+T21MVV
+T21MWM
+T21MWN
+T21MWP
+T21MWQ
+T21MWR
+T21MWS
+T21MWT
+T21MWU
+T21MWV
+T21MXM
+T21MXN
+T21MXP
+T21MXQ
+T21MXR
+T21MXS
+T21MXT
+T21MXU
+T21MXV
+T21MYM
+T21MYN
+T21MYP
+T21MYQ
+T21MYR
+T21MYS
+T21MYT
+T21MYU
+T21MYV
+T21MZM
+T21MZN
+T21MZP
+T21MZQ
+T21MZR
+T21MZS
+T21MZT
+T21MZU
+T21MZV
+T21NTA
+T21NTB
+T21NTC
+T21NTD
+T21NTE
+T21NTF
+T21NTG
+T21NUA
+T21NUB
+T21NUC
+T21NUD
+T21NUE
+T21NUF
+T21NUG
+T21NVA
+T21NVB
+T21NVC
+T21NVD
+T21NVE
+T21NVF
+T21NVG
+T21NWA
+T21NWB
+T21NWC
+T21NWD
+T21NWE
+T21NWF
+T21NWG
+T21NXA
+T21NXB
+T21NXC
+T21NXD
+T21NXE
+T21NXF
+T21NXG
+T21NYA
+T21NYB
+T21NYC
+T21NYD
+T21NYE
+T21NYF
+T21NYG
+T21NZA
+T21NZB
+T21NZC
+T21NZD
+T21NZE
+T21NZF
+T21NZG
+T22LBH
+T22LBJ
+T22LBK
+T22LBL
+T22LBM
+T22LBN
+T22LBP
+T22LBQ
+T22LBR
+T22LCH
+T22LCJ
+T22LCK
+T22LCL
+T22LCM
+T22LCN
+T22LCP
+T22LCQ
+T22LCR
+T22LDH
+T22LDJ
+T22LDK
+T22LDL
+T22LDM
+T22LDN
+T22LDP
+T22LDQ
+T22LDR
+T22LEH
+T22LEJ
+T22LEK
+T22LEL
+T22LEM
+T22LEN
+T22LEP
+T22LEQ
+T22LER
+T22LFH
+T22LFJ
+T22LFK
+T22LFL
+T22LFM
+T22LFN
+T22LFP
+T22LFQ
+T22LFR
+T22LGH
+T22LGJ
+T22LGK
+T22LGL
+T22LGM
+T22LGN
+T22LGP
+T22LGQ
+T22LGR
+T22LHH
+T22LHJ
+T22LHK
+T22LHL
+T22LHM
+T22LHN
+T22LHP
+T22LHQ
+T22LHR
+T22MBA
+T22MBB
+T22MBC
+T22MBD
+T22MBE
+T22MBS
+T22MBT
+T22MBU
+T22MBV
+T22MCA
+T22MCB
+T22MCC
+T22MCD
+T22MCE
+T22MCS
+T22MCT
+T22MCU
+T22MCV
+T22MDA
+T22MDB
+T22MDC
+T22MDD
+T22MDE
+T22MDS
+T22MDT
+T22MDU
+T22MDV
+T22MEA
+T22MEB
+T22MEC
+T22MED
+T22MEE
+T22MES
+T22MET
+T22MEU
+T22MEV
+T22MFA
+T22MFB
+T22MFC
+T22MFD
+T22MFE
+T22MFS
+T22MFT
+T22MFU
+T22MFV
+T22MGA
+T22MGB
+T22MGC
+T22MGD
+T22MGE
+T22MGS
+T22MGT
+T22MGU
+T22MGV
+T22MHA
+T22MHB
+T22MHC
+T22MHD
+T22MHE
+T22MHS
+T22MHT
+T22MHU
+T22MHV
+T22NBF
+T22NBG
+T22NBH
+T22NBJ
+T22NBK
+T22NBL
+T22NBM
+T22NCF
+T22NCG
+T22NCH
+T22NCJ
+T22NCK
+T22NCL
+T22NCM
+T22NDF
+T22NDG
+T22NDH
+T22NDJ
+T22NDK
+T22NDL
+T22NEF
+T22NEG
+T22NEH
+T22NEJ
+T22NFF
+T22NFG
+T22NFH
+T23LKC
+T23LKD
+T23LKE
+T23LKF
+T23LKG
+T23LKH
+T23LKJ
+T23LKK
+T23LKL
+T23LLC
+T23LLD
+T23LLE
+T23LLF
+T23LLG
+T23LLH
+T23LLJ
+T23LLK
+T23LLL
+T23LMC
+T23LMD
+T23LME
+T23LMF
+T23LMG
+T23LMH
+T23LMJ
+T23LMK
+T23LML
+T23MKM
+T23MKN
+T23MKP
+T23MKQ
+T23MKR
+T23MKS
+T23MKT
+T23MKU
+T23MKV
+T23MLM
+T23MLN
+T23MLP
+T23MLQ
+T23MLR
+T23MLS
+T23MLT
+T23MLU
+T23MLV
+T23MMM
+T23MMN
+T23MMP
+T23MMQ
+T23MMR
+T23MMS
+T23MMT
+T23MMU

hls.py

+import os
+import pandas as pd
+import requests
+import rasterio as rio
+import matplotlib.pyplot as plt
+from datetime import datetime
+from shapely.geometry import Polygon
+import xml.etree.ElementTree as ET
+from all_functions import parse_kml_to_tiles
+from all_functions import search_hls_granules_by_tile
+from all_functions import acquisition_download_links_to_csv
+from all_functions import plot_union_polygon
+
+
+
+
+kml_file_path = './hls_tiles.kml'
+output_file_name = "amazon-download-links.csv"
+date_from = "2020-01-01"
+date_to = "2024-12-31"
+cloud_coverage_step = 10
+ifPlot = False
+tile_id_list = list(pd.read_csv("amazon-dlr-tile-ids.csv", header=None)[0])
+total_number_of_tiles = len(tile_id_list)
+# tile_id_list = ["T17LRJ"]
+# tile_id_list = ["T21MYS"]
+tile_index = 0
+if os.path.isfile(output_file_name): os.remove(output_file_name)
+for tile_id in tile_id_list:
+    ifComplete = False
+    tile_index += 1
+    counter_message = f" ({tile_index}/{total_number_of_tiles})."
+    polygon_surface_relative_diff_percent = 100
+    tile_polygon, tile_coords = parse_kml_to_tiles(kml_file_path, target_tile_id=tile_id)
+    # granules = search_hls_granules_by_tile(["HLSL30"], tile_id, date_from, date_to, cloud_coverage_threshold)
+    needed_number_of_acquisitions = 0
+    for cloud_coverage_threshold in range(0,101,cloud_coverage_step):
+        granules = search_hls_granules_by_tile(["HLSL30", "HLSS30"], tile_id, date_from, date_to, cloud_coverage_threshold, cloud_coverage_step)
+        union_polygon = Polygon([])
+        for acquisition in granules:
+            needed_number_of_acquisitions += 1
+            # print(acquisition['title'][15:-5])
+            coord_tmp  = acquisition['polygons'][0][0].split()
+            acquisition_coords = [[float(coord_tmp[i+1]),float(coord_tmp[i])] for i in range(0,int(len(coord_tmp)),2)]
+            acquisition_polygon = Polygon(acquisition_coords)
+            union_polygon = union_polygon.union(acquisition_polygon)
+            union_polygon = tile_polygon.intersection(union_polygon)
+            polygon_surface_relative_diff_percent = (tile_polygon.area - union_polygon.area) / tile_polygon.area*100 
+            if polygon_surface_relative_diff_percent < 1e-1:
+                ifComplete = True
+#                 plot_union_polygon(union_polygon, tile_polygon)
+                break
+        if polygon_surface_relative_diff_percent < 1e-1:
+            status_message = f"{tile_id}, cloud coverage: {cloud_coverage_threshold}, complete in {needed_number_of_acquisitions} acquisitions."
+        else:
+            status_message = f"{tile_id}, cloud coverage: {cloud_coverage_threshold}, incomplete in {needed_number_of_acquisitions} acquisitions."
+        print(status_message, counter_message, end="\r")
+        # if polygon_surface_relative_diff_percent > 1e-1:
+        #     status = f"{tile_id} is incomplete, surface diff: {polygon_surface_relative_diff_percent}%. Number of acquisitions: {len(granules)}."
+        #     if ifPlot:
+        
+        acquisition_download_links_to_csv(tile_id, output_file_name, acquisitions=granules[0:needed_number_of_acquisitions], ifComplete=ifComplete)
+        if ifComplete: 
+            break
+    # break
+
+
+
+
+