From 952817fa2f1d9b5a8a0acb8fd669f52d29938991 Mon Sep 17 00:00:00 2001 From: Carlotta Date: Thu, 23 Jan 2025 15:56:54 +0100 Subject: [PATCH 1/2] updated colorbar --- mapies/mapies.py | 64 +++++------------------------------------------- mapies/viirs.py | 9 +++---- run/run_viirs.py | 17 ++++++------- 3 files changed, 18 insertions(+), 72 deletions(-) diff --git a/mapies/mapies.py b/mapies/mapies.py index 9bde90ce..ba822941 100644 --- a/mapies/mapies.py +++ b/mapies/mapies.py @@ -323,12 +323,13 @@ class MAPIES: levels = np.linspace(unique_values.min(), unique_values.max(), unique_values.max()-unique_values.min()+1) # Use a discrete colormap - cmap = plt.cm.viridis - norm = mcolors.BoundaryNorm(np.arange(unique_values.min()-0.5, unique_values.max()+1, 1), cmap.N) + cmap = plt.cm.rainbow + norm = mcolors.Normalize(vmin=unique_values.min(), vmax=unique_values.max()) + # norm = mcolors.BoundaryNorm(np.arange(unique_values.min()-0.5, unique_values.max()+1, 1), cmap.N) # Create the plot and add features fig, ax = plt.subplots(subplot_kw={"projection": proj}, figsize=figsize) - ax.gridlines() + ax.gridlines(draw_labels=True) ax.coastlines(resolution="10m") # Scatter plot for the number of observations @@ -337,7 +338,7 @@ class MAPIES: ) # Add the discrete colorbar without arrow-like ends - cbar = fig.colorbar(im, ax=ax, orientation="vertical", ticks=levels) + cbar = fig.colorbar(im, ax=ax, orientation="vertical") cbar.set_label("Number of Observations") # cbar.ax.set_yticklabels([f"{level}" for level in range(unique_values)]) @@ -353,57 +354,4 @@ class MAPIES: print(f"Saved plot: {filepath}") - @timeit - def plot_2D_num_obs_custom_log_option( - self, lon_values=None, lat_values=None, num_obs=None, outdir="./", - title=None, filename=None, use_log_scale=False - ): - """ - General method for plotting the number of observations contributing to the final averages, - with an option to use a logarithmic scale for large observation counts. - """ - - # Set Basemap projection - figsize = (15, 10) - markersize = 2.5 - proj = ccrs.PlateCarree() - - # Handle log scale if needed - if use_log_scale: - num_obs = np.log10(num_obs + 1) # Avoid log(0) by adding 1 - label = "Log10(Number of Observations)" - else: - label = "Number of Observations" - - # Define discrete levels for the color bar - unique_values = np.unique(num_obs) - levels = np.linspace(unique_values.min(), unique_values.max(), 10) - - # Use a discrete colormap - cmap = plt.cm.rainbow - norm = mcolors.BoundaryNorm(levels, cmap.N) - - # Create the plot and add features - fig, ax = plt.subplots(subplot_kw={"projection": proj}, figsize=figsize) - ax.gridlines(draw_labels=True) - ax.coastlines(resolution="10m") - - # Scatter plot for the number of observations - im = ax.scatter( - lon_values, lat_values, markersize, c=num_obs, cmap=cmap, norm=norm, transform=proj - ) - - # Add the colorbar - cbar = fig.colorbar(im, ax=ax, orientation="vertical", ticks=levels) - cbar.set_label(label) - - # Add title and save the plot - ax.set_title(title or "Number of Observations Plot") - - # Ensure the output directory exists - os.makedirs(outdir, exist_ok=True) - - filepath = os.path.join(outdir, filename or "num_observation_2D_plot.png") - plt.savefig(filepath, format="png") - plt.close(fig) - print(f"Saved plot: {filepath}") + \ No newline at end of file diff --git a/mapies/viirs.py b/mapies/viirs.py index 16c2f2bd..501ac68f 100644 --- a/mapies/viirs.py +++ b/mapies/viirs.py @@ -455,7 +455,7 @@ class VIIRS(MAPIES): continue # title = f"Observation 2D plot of {self.datatype.upper()} data from {self.start_date} to {self.end_date}" title = f"Observation 2D plot of month {m} from {self.start_date} to {self.end_date}" - filename = f"{outdir}/{self.datatype}_2D_obs_{m}_{self.year}_JANUARY_trial7.png" + filename = f"{outdir}/{self.datatype}_2D_obs_month{m}_{self.year}_.png" super().plot_2D_obs_custom( lon_values=lon_values, lat_values=lat_values, @@ -500,15 +500,14 @@ class VIIRS(MAPIES): continue # title = f"Observation 2D plot of {self.datatype.upper()} data from {self.start_date} to {self.end_date}" title = f"Plot of the number of valid observations for month {m} from {self.start_date} to {self.end_date}" - filename = f"{outdir}/{self.datatype}_2D_obs_count_{m}_{self.year}_JANUARY_trial7.png" - super().plot_2D_num_obs_custom_log_option( + filename = f"{outdir}/{self.datatype}_2D_obs_count_month{m}_{self.year}_trial4.png" + super().plot_2D_num_obs_custom( lon_values=lon_values, lat_values=lat_values, num_obs=cum_count, outdir=outdir, title=title, filename=filename, - use_log_scale=True, ) else: try: @@ -529,7 +528,7 @@ class VIIRS(MAPIES): filename=filename, ) - + @timeit def process_data(self, monthly_avg=True, batch_size=100): """ Process the data for the specified year and months. diff --git a/run/run_viirs.py b/run/run_viirs.py index 09050fc9..8685c6cd 100644 --- a/run/run_viirs.py +++ b/run/run_viirs.py @@ -14,17 +14,16 @@ if __name__ == "__main__": # end_date = "202401010336" # ONE YEAR - start_date = "202401300000" - # end_date = "202412312359" - end_date = "202402022359" + # start_date = "202401300000" + # end_date = "202402022359" # ONE MONTH - JANUARY 2024 - # start_date = "202407010000" - # end_date = "202407312359" + start_date = "202401010000" + end_date = "202401312359" # outdir="/home/cmeikle/Projects/data/" # indir="/home/cmeikle/Projects/data/VIIRS/original_files/AERDB_L2_VIIRS_NOAA20" - outdir = "/home/cgile/Documents/mapies/mapies/figures" + outdir = "/home/cgile/Documents/mapies/figures" # indir = "/home/cgile/bscearth000/esarchive/obs/nasa/viirs_noaa20_aerdb_l2/original_files/VIIRS" indir = "/home/cgile/Documents/mapies/VIIRS" @@ -34,10 +33,10 @@ if __name__ == "__main__": c.read_nc() c.process_data(monthly_avg=True, batch_size = 100) c.yearly_average() - c.plot_2D_observations(months=None, outdir=outdir) - # c.plot_2D_num_obs(months=None, outdir=outdir) + c.plot_2D_observations(months=[1], outdir=outdir) + c.plot_2D_num_obs(months=[1], outdir=outdir) end_time = time.time() elapsed_time = end_time - start_time - # print(f"Script executed in {elapsed_time:.2f} seconds.") + print(f"Script executed in {elapsed_time:.2f} seconds.") -- GitLab From 7258cc6c72f4bab0ba4f4cde0844a80c11692c41 Mon Sep 17 00:00:00 2001 From: Carlotta Date: Thu, 23 Jan 2025 16:43:27 +0100 Subject: [PATCH 2/2] updated mapies.py --- mapies/mapies.py | 86 +++++++++++++++--------------------------------- 1 file changed, 27 insertions(+), 59 deletions(-) diff --git a/mapies/mapies.py b/mapies/mapies.py index e3986061..2129a033 100644 --- a/mapies/mapies.py +++ b/mapies/mapies.py @@ -235,8 +235,33 @@ class MAPIES: # return reindexed values return independent_var_values - - + def quality_assurance(self): + """ + Applying quality assurance flags to mapies data + """ + mask = np.zeros_like(self.obs, dtype=bool) + for i, (flag, info) in enumerate(self.qualityFlags.items()): + + flag_mask = np.zeros_like(self.obs, dtype=bool) + arr = self.ds[info["qa_name"]].values.flatten() + arr = self.reindex(self.time_values_index, arr) + + + for value in info["qa_values"]: + flag_mask |= (arr == value) + if i == 0: + mask = flag_mask + else: + if info["qa_combined"]: + mask &= flag_mask + else: + mask |= flag_mask + + self.obs = self.obs[mask] + self.lon_values = self.lon_values[mask] + self.lat_values = self.lat_values[mask] + + @staticmethod def to_xarray(coords:dict, data_vars:dict, **kwargs): """ @@ -360,62 +385,5 @@ class MAPIES: plt.close(fig) print(f"Saved plot: {filepath}") -<<<<<<< HEAD -======= - @timeit - def plot_2D_num_obs_custom_log_option( - self, lon_values=None, lat_values=None, num_obs=None, outdir="./", - title=None, filename=None, use_log_scale=False - ): - """ - General method for plotting the number of observations contributing to the final averages, - with an option to use a logarithmic scale for large observation counts. - """ - - # Set Basemap projection - figsize = (15, 10) - markersize = 2.5 - proj = ccrs.PlateCarree() - - # Handle log scale if needed - if use_log_scale: - num_obs = np.log10(num_obs + 1) # Avoid log(0) by adding 1 - label = "Log10(Number of Observations)" - else: - label = "Number of Observations" - - # Define discrete levels for the color bar - unique_values = np.unique(num_obs) - levels = np.linspace(unique_values.min(), unique_values.max(), 10) - - # Use a discrete colormap - cmap = plt.cm.rainbow - norm = mcolors.BoundaryNorm(levels, cmap.N) - - # Create the plot and add features - fig, ax = plt.subplots(subplot_kw={"projection": proj}, figsize=figsize) - ax.gridlines(draw_labels=True) - ax.coastlines(resolution="10m") - - # Scatter plot for the number of observations - im = ax.scatter( - lon_values, lat_values, markersize, c=num_obs, cmap=cmap, norm=norm, transform=proj - ) - - # Add the colorbar - cbar = fig.colorbar(im, ax=ax, orientation="vertical", ticks=levels) - cbar.set_label(label) - - # Add title and save the plot - ax.set_title(title or "Number of Observations Plot") - - # Ensure the output directory exists - os.makedirs(outdir, exist_ok=True) - - filepath = os.path.join(outdir, filename or "num_observation_2D_plot.png") - plt.savefig(filepath, format="png") - plt.close(fig) - print(f"Saved plot: {filepath}") ->>>>>>> master -- GitLab