From aad4bbb2d9b43f49ae3825e168c6d6634c8c9551 Mon Sep 17 00:00:00 2001 From: Calum Meikle Date: Thu, 6 Mar 2025 15:19:41 +0100 Subject: [PATCH 1/3] Updated with self.time_orig --- mapies/config/satellite_config.yaml | 21 +++++++++++++++++++++ mapies/mapies.py | 8 +++++--- mapies/tropomi.py | 2 +- run/run_tropomi.py | 6 +++--- run/run_viirs.py | 8 ++++---- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/mapies/config/satellite_config.yaml b/mapies/config/satellite_config.yaml index 36ef5e04..f2d04bb3 100644 --- a/mapies/config/satellite_config.yaml +++ b/mapies/config/satellite_config.yaml @@ -56,4 +56,25 @@ grids: dlat: 0.1 #nlon: 3600 #nlat: 1800 + cams: + centre_lat: 35 + centre_lon: 15 + west: -35 + south: -27 + dlon: 0.2 + dlat: 0.2 + bdrc: + centre_lat: 35 + centre_lon: 20 + west: -51 + south: -35 + dlon: 0.1 + dlat: 0.1 + global: + west: -180 + south: -90 + dlon: 0.1 + dlat: 0.1 + #nlon: 3600 + #nlat: 1800 diff --git a/mapies/mapies.py b/mapies/mapies.py index 0b457768..222d6095 100644 --- a/mapies/mapies.py +++ b/mapies/mapies.py @@ -48,7 +48,8 @@ class MAPIES: "regular":RegularGrid, "ireg_rotated":IrregularRotatedGrid, "cams":CAMSGrid, - "bdrc":BDRCGrid + "bdrc":BDRCGrid, + "global":RegularGrid } @@ -363,14 +364,15 @@ class MAPIES: # Convert time to seconds since TIME_ORIGIN final_time_seconds = (time - self.time_orig) / np.timedelta64(1, "s") final_time_seconds = np.round(final_time_seconds, 0) - + grid_representation_str = json.dumps(self.grid_config, ensure_ascii=False) qa_flags_str = json.dumps(self.qualityFlags, ensure_ascii=False) + # Create xarray Dataset ds = xr.Dataset( coords={"time": ("time", final_time_seconds, { - "units": "seconds since 1900-01-01 00:00:00", + "units": f"seconds since {self.time_orig}", "calendar": "gregorian" })}, diff --git a/mapies/tropomi.py b/mapies/tropomi.py index 180649c8..fe13be38 100644 --- a/mapies/tropomi.py +++ b/mapies/tropomi.py @@ -31,7 +31,7 @@ class TROPOMI(MAPIES): """ super().__init__(start_date, end_date) - self.time_orig = np.datetime64("1900-01-01T00:00:00") + self.time_orig = np.datetime64("1993-01-01T00:00:00") self.dest = kwargs.get("dest") if not os.path.exists(self.dest): raise Exception("Output directory doesn't exist") diff --git a/run/run_tropomi.py b/run/run_tropomi.py index fd74d50e..c9bdf577 100644 --- a/run/run_tropomi.py +++ b/run/run_tropomi.py @@ -12,13 +12,13 @@ if __name__ == "__main__": start_time = time.time() - c = TROPOMI(start_date, end_date, dest=outdir, indir=indir, grid_repr="bdrc") + c = TROPOMI(start_date, end_date, dest=outdir, indir=indir, grid_repr="global") c.gather_nc_files() - c.process_avg_data(monthly_avg=False, batch_size = 14, apply_qa=True, geospatial_crop=[[-40, 60], [0, 70]]) + c.process_avg_data(monthly_avg=False, batch_size = 14, apply_qa=True, geospatial_crop=[[-40, 60], [0, 70]], save=True) c.plot_2D_observations(months=[0], outdir=outdir) c.plot_2D_num_obs(months=[0], outdir=outdir) - c.process_lazy_data(apply_qa=True, geospatial_crop=[[-40, 60], [0, 70]]) + #c.process_lazy_data(apply_qa=True, geospatial_crop=[[-40, 60], [0, 70]]) end_time = time.time() elapsed_time = end_time - start_time print(f"Script executed in {elapsed_time:.2f} seconds.") diff --git a/run/run_viirs.py b/run/run_viirs.py index d4ced0cd..31587112 100644 --- a/run/run_viirs.py +++ b/run/run_viirs.py @@ -19,14 +19,14 @@ if __name__ == "__main__": start_time = time.time() - c = VIIRS(start_date, end_date, dest=outdir, indir=indir, grid_repr="cams") + c = VIIRS(start_date, end_date, dest=outdir, indir=indir, grid_repr="rotated") c.gather_nc_files() - c.process_avg_data(monthly_avg=True, batch_size = 100, apply_qa=True, save=True) + c.process_avg_data(monthly_avg=False, batch_size = 100, apply_qa=True, save=True) c.yearly_average() - c.plot_2D_observations(months=None, filename="new.png", outdir=outdir) - c.plot_2D_num_obs(months=None, outdir=outdir) + c.plot_2D_observations(months=[0], filename="new.png", outdir=outdir) + c.plot_2D_num_obs(months=[0], outdir=outdir) c.process_lazy_data(apply_qa=True, save=False) c.to_da(frequency="3H", save_figs=True) -- GitLab From 0736d3136d6f8ebcf7d98ec990ea158c71357eb6 Mon Sep 17 00:00:00 2001 From: Calum Meikle Date: Fri, 7 Mar 2025 11:22:58 +0100 Subject: [PATCH 2/3] Updated gitignore --- .gitignore | 2 +- run/run_viirs.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 9faf17c8..7cef2fe6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,8 @@ outputs __pycache__ /mapies.egg-info/ /build/ -run/run-nord3.sh run/run_viirs.py +run/run_tropomi.py grids/__pycache__ util/__pycache__ tests/__pycache__ diff --git a/run/run_viirs.py b/run/run_viirs.py index 31587112..fe1a2fee 100644 --- a/run/run_viirs.py +++ b/run/run_viirs.py @@ -19,12 +19,12 @@ if __name__ == "__main__": start_time = time.time() - c = VIIRS(start_date, end_date, dest=outdir, indir=indir, grid_repr="rotated") + c = VIIRS(start_date, end_date, dest=outdir, indir=indir, grid_repr="cams") c.gather_nc_files() c.process_avg_data(monthly_avg=False, batch_size = 100, apply_qa=True, save=True) - c.yearly_average() + #c.yearly_average() c.plot_2D_observations(months=[0], filename="new.png", outdir=outdir) c.plot_2D_num_obs(months=[0], outdir=outdir) c.process_lazy_data(apply_qa=True, save=False) -- GitLab From 38152a5260cb2d14508e4ff244763021fc5cd1d9 Mon Sep 17 00:00:00 2001 From: Calum Meikle Date: Fri, 7 Mar 2025 11:31:06 +0100 Subject: [PATCH 3/3] Update version --- CHANGELOG | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 74ed4172..07782405 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,11 @@ CHANGELOG +## 0.1.2 -2025/03/07 - BUG time in netcdf outputs + +BUG FIX + +### Features +- Outputted time was incorrect in the netcdfs with some viirs data saying 21931, this has been fixed ## 0.1.1 - 2025/02/28 - Code refactoring diff --git a/setup.py b/setup.py index de6cc56e..18fbe5e7 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ from setuptools import find_packages from setuptools import setup # Could update this using versioneer -version="0.1.1" +version="0.1.2" setup( name="mapies", -- GitLab