diff --git a/README.md b/README.md index 3721c14..7b58ec2 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,6 @@ Set the parameters in the file [`parameters.json`](input/parameters.json): - The path into the `senet_folder` variable. The `senet_folder` is something like `"C:\\Users\\user\\.snap\\auxdata\\sen-et-conda-Win64"`. - It is also possible to modify all the default computational parameters, as explained in Section 3.3 of the official guide. The *only* parameter that should be modified is the [timezone of the area](input/parameters.json#L12). -Finally, copy the absolute path of all the Sentinel Images in the files [`s2_paths.txt`](input/s2_paths.txt) and [`s3_paths.txt`](input/s3_paths.txt). The path should point to the folders `.SAFE` for S2 and `.SEN3` for S3. - ## Run the code In a terminal run (for Windows use the Git bash terminal): ``` diff --git a/assets/new_full.png b/assets/new_full.png index bdd4af0..492efa1 100644 Binary files a/assets/new_full.png and b/assets/new_full.png differ diff --git a/assets/old.png b/assets/old.png index ce2b8ac..79debf6 100644 Binary files a/assets/old.png and b/assets/old.png differ diff --git a/graph/BiophysicalOp_sentinel_2_pre_processing.xml b/graph/BiophysicalOp_sentinel_2_pre_processing.xml deleted file mode 100644 index ff5163e..0000000 --- a/graph/BiophysicalOp_sentinel_2_pre_processing.xml +++ /dev/null @@ -1,87 +0,0 @@ - - 1.0 - - Read - - - !INPUT_Sentinel-2_L2A! - - - - Resample - - - - - - - - 20 - Nearest - First - First - - - true - - - - Subset - - - - - B2,B3,B4,B5,B6,B7,B8A,B11,B12,quality_aot,quality_wvp,quality_cloud_confidence,quality_snow_confidence,quality_scene_classification,view_zenith_mean,view_azimuth_mean,sun_zenith,sun_azimuth,view_zenith_B1,view_azimuth_B1,view_zenith_B2,view_azimuth_B2,view_zenith_B3,view_azimuth_B3,view_zenith_B4,view_azimuth_B4,view_zenith_B5,view_azimuth_B5,view_zenith_B6,view_azimuth_B6,view_zenith_B7,view_azimuth_B7,view_zenith_B8,view_azimuth_B8,view_zenith_B8A,view_azimuth_B8A,view_zenith_B9,view_azimuth_B9,view_zenith_B10,view_azimuth_B10,view_zenith_B11,view_azimuth_B11,view_zenith_B12,view_azimuth_B12 - 0,0,5490,5490 - - - 1 - 1 - false - - true - - - - BiophysicalOp - - - - - !INPUT_Sentinel-2_Sensor! - true - true - true - true - true - - - - Write - - - - - !OUTPUT_biophysical! - BEAM-DIMAP - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/graph/reflectance_sentinel_2_pre_processing.xml b/graph/reflectance_sentinel_2_pre_processing.xml deleted file mode 100644 index ea85e48..0000000 --- a/graph/reflectance_sentinel_2_pre_processing.xml +++ /dev/null @@ -1,90 +0,0 @@ - - 1.0 - - Read - - - !INPUT_Sentinel-2_L2A! - - - - Resample - - - - - - - - 20 - Nearest - First - First - - - true - - - - Subset - - - - - B2,B3,B4,B5,B6,B7,B8A,B11,B12,quality_aot,quality_wvp,quality_cloud_confidence,quality_snow_confidence,quality_scene_classification,view_zenith_mean,view_azimuth_mean,sun_zenith,sun_azimuth,view_zenith_B1,view_azimuth_B1,view_zenith_B2,view_azimuth_B2,view_zenith_B3,view_azimuth_B3,view_zenith_B4,view_azimuth_B4,view_zenith_B5,view_azimuth_B5,view_zenith_B6,view_azimuth_B6,view_zenith_B7,view_azimuth_B7,view_zenith_B8,view_azimuth_B8,view_zenith_B8A,view_azimuth_B8A,view_zenith_B9,view_azimuth_B9,view_zenith_B10,view_azimuth_B10,view_zenith_B11,view_azimuth_B11,view_zenith_B12,view_azimuth_B12 - 0,0,5490,5490 - B2 - - 1 - 1 - false - - true - - - - Write - - - - - !OUTPUT_reflectance! - BEAM-DIMAP - - - - Subset - - - - - B2,B3,B4,B5,B6,B7,B8A,B11,B12 - 0,0,5490,5490 - B2 - - 1 - 1 - false - - true - - - - - - - - - - - - - - - - - - - - - diff --git a/graph/resample_sentinel_2_pre_processing.xml b/graph/resample_sentinel_2_pre_processing.xml deleted file mode 100644 index b1c9027..0000000 --- a/graph/resample_sentinel_2_pre_processing.xml +++ /dev/null @@ -1,50 +0,0 @@ - - 1.0 - - Read - - - !INPUT_Sentinel-2_L2A! - - - - Resample - - - - - - - - 20 - Nearest - First - First - - - true - - - - Write - - - - - !OUTPUT_resampled! - BEAM-DIMAP - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/graph/BandMaths_mask_sentinel_2_pre_processing.xml b/graph/sentinel_2_pre_processing.xml similarity index 51% rename from graph/BandMaths_mask_sentinel_2_pre_processing.xml rename to graph/sentinel_2_pre_processing.xml index a015d5b..ebf9073 100644 --- a/graph/BandMaths_mask_sentinel_2_pre_processing.xml +++ b/graph/sentinel_2_pre_processing.xml @@ -42,6 +42,74 @@ true + + BiophysicalOp + + + + + !INPUT_Sensor_S2! + true + true + true + true + true + + + + Write + + + + + !OUTPUT_reflectance! + BEAM-DIMAP + + + + Subset + + + + + B2,B3,B4,B5,B6,B7,B8A,B11,B12 + 0,0,5490,5490 + B2 + + 1 + 1 + false + + true + + + + Subset + + + + + sun_zenith + 0,0,5490,5490 + B2 + + 1 + 1 + false + + true + + + + Write + + + + + !OUTPUT_sun_zenith_angle! + BEAM-DIMAP + + BandMaths @@ -71,10 +139,20 @@ BEAM-DIMAP + + Write + + + + + !OUTPUT_biophysical! + BEAM-DIMAP + + - + @@ -82,11 +160,29 @@ + + + + + + + + + + + + + + + + + + diff --git a/graph/sentinel_3_cut.xml b/graph/sentinel_3_cut.xml deleted file mode 100644 index 68de3d8..0000000 --- a/graph/sentinel_3_cut.xml +++ /dev/null @@ -1,49 +0,0 @@ - - 1.0 - - Read - - - !INPUT_Sentinel-3_LST! - - - - Subset - - - - - NDVI,TCWV,biome,fraction,LST,LST_uncertainty,exception,x_in,y_in,bayes_in,cloud_in,confidence_in,pointing_in,probability_cloud_dual_in,probability_cloud_single_in,elevation_in,latitude_in,longitude_in,detector_in,pixel_in,scan_in - - - !INPUT_AOI_WKT! - 1 - 1 - false - - true - - - - Write - - - - - !OUTPUT_Sentinel-3_cut! - BEAM-DIMAP - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/graph/sentinel_3_pre_processing_fix.xml b/graph/sentinel_3_pre_processing.xml similarity index 92% rename from graph/sentinel_3_pre_processing_fix.xml rename to graph/sentinel_3_pre_processing.xml index 877669d..0134a66 100644 --- a/graph/sentinel_3_pre_processing_fix.xml +++ b/graph/sentinel_3_pre_processing.xml @@ -48,13 +48,7 @@ - GEOGCS["WGS84(DD)", - DATUM["WGS84", - SPHEROID["WGS84", 6378137.0, 298.257223563]], - PRIMEM["Greenwich", 0.0], - UNIT["degree", 0.017453292519943295], - AXIS["Geodetic longitude", EAST], - AXIS["Geodetic latitude", NORTH]] + EPSG:4326 Nearest @@ -160,7 +154,7 @@ - + @@ -187,7 +181,7 @@ - + - \ No newline at end of file + diff --git a/graph/sun_zenith_sentinel_2_pre_processing.xml b/graph/sun_zenith_sentinel_2_pre_processing.xml deleted file mode 100644 index ce0febb..0000000 --- a/graph/sun_zenith_sentinel_2_pre_processing.xml +++ /dev/null @@ -1,90 +0,0 @@ - - 1.0 - - Read - - - !INPUT_Sentinel-2_L2A! - - - - Resample - - - - - - - - 20 - Nearest - First - First - - - true - - - - Subset - - - - - B2,B3,B4,B5,B6,B7,B8A,B11,B12,quality_aot,quality_wvp,quality_cloud_confidence,quality_snow_confidence,quality_scene_classification,view_zenith_mean,view_azimuth_mean,sun_zenith,sun_azimuth,view_zenith_B1,view_azimuth_B1,view_zenith_B2,view_azimuth_B2,view_zenith_B3,view_azimuth_B3,view_zenith_B4,view_azimuth_B4,view_zenith_B5,view_azimuth_B5,view_zenith_B6,view_azimuth_B6,view_zenith_B7,view_azimuth_B7,view_zenith_B8,view_azimuth_B8,view_zenith_B8A,view_azimuth_B8A,view_zenith_B9,view_azimuth_B9,view_zenith_B10,view_azimuth_B10,view_zenith_B11,view_azimuth_B11,view_zenith_B12,view_azimuth_B12 - 0,0,5490,5490 - B2 - - 1 - 1 - false - - true - - - - Subset - - - - - sun_zenith - 0,0,5490,5490 - B2 - - 1 - 1 - false - - true - - - - Write - - - - - !OUTPUT_sun_zenith_angle! - BEAM-DIMAP - - - - - - - - - - - - - - - - - - - - - diff --git a/input/parameters.json b/input/parameters.json index a8641ad..5a1233a 100644 --- a/input/parameters.json +++ b/input/parameters.json @@ -1,12 +1,12 @@ { "AOI":{ - "north": 41.6, - "south": 40.4, - "west": 14.9, - "est": 16.5 + "north": 46, + "south": 44, + "west": 6, + "est": 8 }, - "general_path": "D:\\TEMP\\", - "senet_folder": "C:\\Users\\user\\.snap\\auxdata\\sen-et-conda-Win64", + "general_path": "E:\\TEMP\\", + "senet_folder": "C:\\Users\\stucchi\\.snap\\auxdata\\sen-et-conda-Win64", "comp_parameters":{ "ecwf_data":{ "time_zone": 2 diff --git a/main.sh b/main.sh index f0bd7cf..bb5ef64 100644 --- a/main.sh +++ b/main.sh @@ -1,10 +1,18 @@ #!/bin/bash -python py_scripts/1_create_script_graph.py -sh output/1_script_gpt_s3.sh -python py_scripts/2_read_times_s3.py -sh output/2_script_gpt_s2.sh -python py_scripts/3_create_script_python.py -sh output/3_script_S2.sh -sh output/4_script_S3.sh -read -r -p "Computation ended sucessfully! Press any key to close the panel" input \ No newline at end of file +time python py_scripts/0_path_read.py +echo "===================" +time python py_scripts/1_create_script_graph.py +echo "===================" +time sh output/1_script_gpt_s3.sh +echo "===================" +time python py_scripts/2_read_times_s3.py +echo "===================" +time sh output/2_script_gpt_s2.sh +echo "===================" +time python py_scripts/3_create_script_python.py +echo "===================" +time sh output/3_script_S2.sh +echo "===================" +time sh output/4_script_S3.sh +#read -r -p "Computation ended sucessfully! Press any key to close the panel" input \ No newline at end of file diff --git a/py_scripts/0_path_read.py b/py_scripts/0_path_read.py new file mode 100644 index 0000000..6b8ceae --- /dev/null +++ b/py_scripts/0_path_read.py @@ -0,0 +1,49 @@ +import os +import json +import datetime + +def is_valid_date(input_date): + try: + date = input_date.split("_") + newDate = datetime.datetime(int(date[0]),int(date[1]),int(date[2])) + return True + except ValueError: + return False + +path_param = "input\\parameters.json" + +with open(path_param, "r") as f: + param = json.load(f) + +general_path = param["general_path"] + +sub_folders = [name for name in os.listdir(general_path) if os.path.isdir(os.path.join(general_path, name))] + +for folder in sub_folders: + temp_path = general_path + folder + if folder == "S2": + list_s2_days = [temp_path + "\\" + name for name in os.listdir(temp_path) if os.path.isdir(os.path.join(temp_path, name))] + valid_s2_days = [day for day in list_s2_days if is_valid_date(day.split("\\")[-1])] + list_s2_images = [[path + "\\" + name for name in os.listdir(path) if os.path.isdir(os.path.join(path, name))] for path in valid_s2_days] + list_s2_images = [item for sublist in list_s2_images for item in sublist] + valid_list_s2_images = [imm for imm in list_s2_images if imm[-5:] == ".SAFE"] + elif folder == "S3": + list_s3_days = [temp_path + "\\" + name for name in os.listdir(temp_path) if os.path.isdir(os.path.join(temp_path, name))] + valid_s3_days = [day for day in list_s3_days if is_valid_date(day.split("\\")[-1])] + list_s3_images = [[path + "\\" + name for name in os.listdir(path) if os.path.isdir(os.path.join(path, name))] for path in valid_s3_days] + list_s3_images = [item for sublist in list_s3_images for item in sublist] + valid_list_s3_images = [imm for imm in list_s3_images if imm[-5:] == ".SEN3"] + +s2_paths = "input/s2_paths.txt" +f_s2_paths = open(s2_paths, "w") +for element in valid_list_s2_images: + f_s2_paths.write(element + "\n") +f_s2_paths.close() + +s3_paths = "input/s3_paths.txt" +f_s3_paths = open(s3_paths, "w") +for element in valid_list_s3_images: + f_s3_paths.write(element + "\n") +f_s3_paths.close() + +print("\tPath files created!\n\t" + s2_paths + "\n\t" + s3_paths) \ No newline at end of file diff --git a/py_scripts/1_create_script_graph.py b/py_scripts/1_create_script_graph.py index b6f87c3..566a043 100644 --- a/py_scripts/1_create_script_graph.py +++ b/py_scripts/1_create_script_graph.py @@ -3,15 +3,10 @@ path_param = "input\\parameters.json" path_file_path_s2 = "input\\s2_paths.txt" path_file_path_s3 = "input\\s3_paths.txt" -path_bandmask = "graph\\BandMaths_mask_sentinel_2_pre_processing.xml" -path_bio = "graph\\BiophysicalOp_sentinel_2_pre_processing.xml" -path_refl = "graph\\reflectance_sentinel_2_pre_processing.xml" -path_resample = "graph\\resample_sentinel_2_pre_processing.xml" -path_zen = "graph\\sun_zenith_sentinel_2_pre_processing.xml" +path_s2_pro = "graph\\sentinel_2_pre_processing.xml" path_LC = "graph\\add_landcover.xml" path_ele = "graph\\add_elevation.xml" -path_cut = "graph\\sentinel_3_cut.xml" -path_s3_pro = "graph\\sentinel_3_pre_processing_fix.xml" +path_s3_pro = "graph\\sentinel_3_pre_processing.xml" path_graph = "output\\graph\\" path_date_s2 = "output\\days_s2.txt" path_date_s3 = "output\\days_s3.txt" @@ -33,20 +28,8 @@ general_path = param["general_path"] # Save xml files to variable -f = open(path_bandmask, "r") -graph_bandmask = f.read() -f.close() -f = open(path_bio, "r") -graph_bio = f.read() -f.close() -f = open(path_refl, "r") -graph_refl = f.read() -f.close() -f = open(path_resample, "r") -graph_resample = f.read() -f.close() -f = open(path_zen, "r") -graph_zen = f.read() +f = open(path_s2_pro, "r") +graph_s2_pro = f.read() f.close() f = open(path_LC, "r") graph_LC = f.read() @@ -54,9 +37,6 @@ f = open(path_ele, "r") graph_ele = f.read() f.close() -f = open(path_cut, "r") -graph_cut = f.read() -f.close() f = open(path_s3_pro, "r") graph_s3_pro = f.read() f.close() @@ -72,24 +52,14 @@ date_s3 = date_s3 + date + "\n" s3_image_path = s3_image_path + "\\xfdumanifest.xml" - - # Cut - script_text = script_text + "echo \"\t Cut the image S3 " + date + "\"\n" - cut_image = path + "\\S3_" + date + "_cut.dim" - text_cut = graph_cut.replace("!INPUT_Sentinel-3_LST!", s3_image_path).replace("!INPUT_AOI_WKT!", AOI_WTK).replace("!OUTPUT_Sentinel-3_cut!", cut_image) - path_cut = path_graph + "sentinel_3_cut_" + date +".xml" - script_text = script_text + "gpt " + path_cut + "\n" - f = open(path_cut, "w") - f.write(text_cut) - f.close() # Processing script_text = script_text + "echo \"\t Processing the image S3 " + date + "\"\n" obs_image = path + "\\S3_" + date + "_obs_geometry.dim" mask_image = path + "\\S3_" + date + "_mask.dim" lst_image = path + "\\S3_" + date + "_lst.dim" - text_s3_pro = graph_s3_pro.replace("!INPUT_Sentinel-3_LST!", cut_image).replace("!INPUT_AOI_WKT!", AOI_WTK).replace("!OUTPUT_observation_geometry!", obs_image).replace("!OUTPUT_mask!", mask_image).replace("!OUTPUT_LST!", lst_image) - path_s3_pro = path_graph + "sentinel_3_preprocessing_fix_" + date +".xml" + text_s3_pro = graph_s3_pro.replace("!INPUT_Sentinel-3_LST!", s3_image_path).replace("!INPUT_AOI_WKT!", AOI_WTK).replace("!OUTPUT_observation_geometry!", obs_image).replace("!OUTPUT_mask!", mask_image).replace("!OUTPUT_LST!", lst_image) + path_s3_pro = path_graph + "sentinel_3_preprocessing_" + date +".xml" script_text = script_text + "gpt " + path_s3_pro + "\n" + "\n\n" f = open(path_s3_pro, "w") f.write(text_s3_pro) @@ -117,55 +87,18 @@ sensor_s2 = " ".join(s2_image_path.split("\\")[-1:])[:3] s2_image_path = s2_image_path + "\\MTD_MSIL2A.xml" - - # Mask file - script_text = script_text + "echo \"\t Computing the mask for the image S2 " + date + "\"\n" - mask_image = path + "\\S2_" + date + "_mask.dim" - text_mask = graph_bandmask.replace("!INPUT_Sentinel-2_L2A!", s2_image_path).replace("!OUTPUT_mask!", mask_image) - path_mask = path_graph + "BandMaths_mask_sentinel_2_pre_processing_" + date +".xml" - script_text = script_text + "gpt " + path_mask + "\n" - f = open(path_mask, "w") - f.write(text_mask) - f.close() - - # Biophysical - script_text = script_text + "echo \"\t Computing the biophysical parameters for the image S2 " + date + "\"\n" + + # Processing + script_text = script_text + "echo \"\t Processing the image S2 " + date + "\"\n" bio_image = path + "\\S2_" + date + "_biophysical.dim" - text_bio = graph_bio.replace("!INPUT_Sentinel-2_L2A!", s2_image_path).replace("!OUTPUT_biophysical!", bio_image).replace("!INPUT_Sentinel-2_Sensor!", sensor_s2) - path_bio = path_graph + "BiophysicalOp_sentinel_2_pre_processing_" + date +".xml" - script_text = script_text + "gpt " + path_bio + "\n" - f = open(path_bio, "w") - f.write(text_bio) - f.close() - - # Reflectance - script_text = script_text + "echo \"\t Computing the reflectance for the image S2 " + date + "\"\n" refl_image = path + "\\S2_" + date + "_reflectance.dim" - text_refl = graph_refl.replace("!INPUT_Sentinel-2_L2A!", s2_image_path).replace("!OUTPUT_reflectance!", refl_image) - path_refl = path_graph + "reflectance_sentinel_2_pre_processing_" + date +".xml" - script_text = script_text + "gpt " + path_refl + "\n" - f = open(path_refl, "w") - f.write(text_refl) - f.close() - - # Resample - script_text = script_text + "echo \"\t Resampling the image S2 " + date + "\"\n" resample_image = path + "\\S2_" + date + "_resampled.dim" - text_resample = graph_resample.replace("!INPUT_Sentinel-2_L2A!", s2_image_path).replace("!OUTPUT_resampled!", resample_image) - path_resample = path_graph + "resample_sentinel_2_pre_processing_" + date +".xml" - script_text = script_text + "gpt " + path_resample + "\n" - f = open(path_resample, "w") - f.write(text_resample) - f.close() - - # Sun Zenith - script_text = script_text + "echo \"\t Computing the sun zenith angular map for the image S2 " + date + "\"\n" zen_image = path + "\\S2_" + date + "_sun_zenith_angle.dim" - text_zen = graph_zen.replace("!INPUT_Sentinel-2_L2A!", s2_image_path).replace("!OUTPUT_sun_zenith_angle!", zen_image) - path_zen = path_graph + "sun_zenith_sentinel_2_pre_processing_" + date +".xml" - script_text = script_text + "gpt " + path_zen + "\n" - f = open(path_zen, "w") - f.write(text_zen) + text_s2_pro = graph_s2_pro.replace("!INPUT_Sentinel-2_L2A!", s2_image_path).replace("!OUTPUT_mask!", mask_image).replace("!OUTPUT_biophysical!", bio_image).replace("!INPUT_Sensor_S2!", sensor_s2).replace("!OUTPUT_reflectance!", refl_image).replace("!OUTPUT_resampled!", resample_image).replace("!OUTPUT_sun_zenith_angle!", zen_image) + path_s2_pro = path_graph + "sentinel_2_pre_processing_" + date +".xml" + script_text = script_text + "gpt " + path_s2_pro + "\n" + f = open(path_s2_pro, "w") + f.write(text_s2_pro) f.close() # Landcover