ecmwf: avoid memory issue by splitting ens grib file
1 files changed, 19 insertions(+), 14 deletions(-)

M meteo_scraper/ecmwf_scraper.py
M meteo_scraper/ecmwf_scraper.py +19 -14
@@ 39,6 39,8 @@ PARAMETERS = {
     'total_precipiation': ('tp', 'kg.m-2'),
 }
 
+PARAMS = ['2t', '10u', '10v', 'tp']
+
 ZONES = [
     'AT', 'BE', 'BG', 'CH', 'CZ', 'DE_LU', 'DK_1', 'DK_2',
     'EE', 'ES', 'FI', 'FR', 'GR', 'HR', 'HU', 'IT_CALA', 'IT_CNOR',

          
@@ 58,17 60,19 @@ def find_last_run(model):
 
 
 def get_ens_data(filename_ens, date, time):
-    client.retrieve(
-        date=date,
-        time=time,
-        step=STEPS[('ENS', time)],
-        stream="enfo",
-        type=['cf', 'pf'],
-        levtype="sfc",
-        param=['2t', '10u', '10v', 'tp'],
-        target=filename_ens
-    )
-    print(f'{filename_ens} file is saved')
+    for param in PARAMS:
+        filename_param = f'{param}_{filename_ens}' 
+        client.retrieve(
+            date=date,
+            time=time,
+            step=STEPS[('ENS', time)],
+            stream="enfo",
+            type=['cf', 'pf'],
+            levtype="sfc",
+            param=[param],
+            target=filename_param
+        )
+        print(f'{filename_param} file is saved')
 
 
 def get_hres_data(filename_hres, date, time):

          
@@ 79,7 83,7 @@ def get_hres_data(filename_hres, date, t
         stream="oper",
         type=['fc'],
         levtype="sfc",
-        param=['2t', '10u', '10v', 'tp'],
+        param=PARAMS,
         target=filename_hres
     )
     print(f'{filename_hres} file is saved')

          
@@ 169,5 173,6 @@ def hres_ecmwfdata_to_refinery(tsa, file
 
 
 def ens_ecmwfdata_to_refinery(tsa, filename_ens):
-    gribfile_ens = mv.read(str(filename_ens))
-    exploit_grib_file_ens(tsa, gribfile_ens)
+    for param in PARAMS:
+        gribfile_ens = mv.read(f'{param}_{filename_ens}')
+        exploit_grib_file_ens(tsa, gribfile_ens)