diff --git a/mapies/viirs.py b/mapies/viirs.py index 65d95b6b6e8c614d72546257404b09a31d558539..b686bdaa952860817488e2323a14f67e9ce20419 100644 --- a/mapies/viirs.py +++ b/mapies/viirs.py @@ -81,11 +81,8 @@ def process_single_file( ds.close() return obs, lat, lon, time_values - - - except Exception as e: - print(f"Error processing file {file}: {e}") + print(f"Error processing file in process single file {file}: {e}") return None @@ -309,7 +306,17 @@ class VIIRS(MAPIES): # self.count_obs = cumulative_count if save: - self.to_netCDF(self.obs, self.lat, self.lon, self.start_date, self.end_date) + try: + mid_time = self.start_date + (self.end_date - self.start_date) / 2 + mid_time = np.datetime64(mid_time) + N = len(self.obs) + mid_time_array = np.repeat(mid_time, N) + + self.to_netCDF(self.obs, self.lat, self.lon, mid_time_array, self.start_date, self.end_date) + print(f'Saved processed data to netCDF') + except Exception as e: + print(f"Error saving data into netCDF: {e}") + return except Exception as e: print(f"Error processing data for the time period from {self.start_date} to {self.end_date}: {e}") @@ -368,6 +375,7 @@ class VIIRS(MAPIES): # Filter valid results and log failed files valid_results = [res for res in results if res is not None] + print(f"Valid results: {len(valid_results)}") failed_files.extend([file for file, res in zip(batch, results) if res is None]) if not valid_results: @@ -469,7 +477,7 @@ class VIIRS(MAPIES): print("Yearly average computation completed.") @timeit - def process_lazy_data(self, apply_qa=True, geospatial_crop: NDArray | None = None): + def process_lazy_data(self, apply_qa=False, geospatial_crop: NDArray | None = None): """ Process the data for the specified time range. It only retrieves a daily dictionary using dask lazy loading. """ @@ -477,6 +485,8 @@ class VIIRS(MAPIES): #Check if the user wants qa flags if apply_qa: self.apply_qa = True + else: + self.apply_qa = False if geospatial_crop is not None: self.geospatial_crop = np.array(geospatial_crop) for day in self.daily_dict.keys(): @@ -508,8 +518,12 @@ class VIIRS(MAPIES): with multiprocessing.Pool(processes=8) as pool: # Adjust number of processes as needed results = pool.starmap(process_single_file, args) - - for obs, lat, lon, time_values in results: + + valid_results = [res for res in results if res is not None] + # Check shapes of obs lat and lon to see if they are all the same + + for obs, lat, lon, time_values in valid_results: + print(f'Shape of obs: {obs.shape}, Shape of lat: {lat.shape}, Shape of lon: {lon.shape}, Shape of time: {time_values.shape}') if len(obs) > 0: daily_lon.append(lon) daily_lat.append(lat) @@ -533,7 +547,6 @@ class VIIRS(MAPIES): except Exception as e: print(f"Error processing data for day {day}: {e}") continue - self.to_netCDF(final_obs, final_lat, final_lon, final_time, self.start_date, self.end_date) def to_netCDF(self, obs, lat, lon, time, start_time, end_time): @@ -562,7 +575,7 @@ class VIIRS(MAPIES): "institution": "Barcelona Supercomputing Center", "grid": f"{self.grid_repr}: {grid_representation_str}", "Developers": "MAPIES team", - "QA": f"Quality assurance applied: {qa_flags_str}", + "QA": f"Quality Assurance: {self.apply_qa}, quality assurance applied: {qa_flags_str}", "history": f"Created {datetime.now()}", }, ) diff --git a/run/run_viirs.py b/run/run_viirs.py index 04967a55fc83385311ac369da546b7150dd27a04..b81ead6e9c60deac9430e21335f14cf1589a450c 100644 --- a/run/run_viirs.py +++ b/run/run_viirs.py @@ -7,14 +7,15 @@ if __name__ == "__main__": # Change these variables # 00:30 to 02:36 from 2024-01-01 to 2024-01-03 start_date = "202401010030" - end_date = "202401030236" + end_date = "202401012359" - outdir="/home/cmeikle/Projects/data/" - indir="/home/cmeikle/Projects/data/VIIRS" - + # outdir="/home/cmeikle/Projects/data/" + # indir="/home/cmeikle/Projects/data/VIIRS" + outdir="/home/cgile/Documents/mapies/figures" + indir="/home/cgile/Documents/mapies/VIIRS" @@ -23,11 +24,11 @@ if __name__ == "__main__": c = VIIRS(start_date, end_date, dest=outdir, indir=indir, grid_repr="rotated") c.read_nc() - c.process_data(monthly_avg=False, batch_size = 50, apply_qa=False, geospatial_crop=[[0, 20], [10, 20]]) + # c.process_data(monthly_avg=False, batch_size = 50, save=True, apply_qa=False) # c.yearly_average() - c.plot_2D_observations(months=[0], filename="Viirs_qa_not_applied.png", outdir=outdir) - c.plot_2D_num_obs(months=[0], filename="Viirs_num_obsqa_not_applied.png", outdir=outdir) - # c.process_lazy_data() + # c.plot_2D_observations(months=[0], filename="Viirs_qa_not_applied.png", outdir=outdir) + # c.plot_2D_num_obs(months=[0], filename="Viirs_num_obsqa_not_applied.png", outdir=outdir) + c.process_lazy_data(apply_qa=False, geospatial_crop=None) end_time = time.time() elapsed_time = end_time - start_time