### Grib2Message Initialization and Properties Source: https://noaa-mdl.github.io/grib2io Handles post-initialization setup and provides property accessors for GRIB2 metadata and message type identification. ```python def __post_init__(self): """Set some attributes after init.""" self._auto_nans = _AUTO_NANS self._coordlist = np.zeros((0), dtype=np.float32) self._data = None self._deflist = np.zeros((0), dtype=np.int64) self._msgnum = -1 self._ondiskarray = None self._orig_section5 = np.copy(self.section5) self._signature = self._generate_signature() try: self._sha1_section3 = hashlib.sha1(self.section3).hexdigest() except TypeError: pass self.bitMapFlag = templates.Grib2Metadata(self.bitMapFlag, table="6.0") self.bitmap = None @property def _isNDFD(self): """Check if GRIB2 message is from NWS NDFD""" return np.all(self.section1[0:2] == [8, 65535]) @property def _isAerosol(self): """Check if GRIB2 message contains aerosol data""" is_aero_template = self.productDefinitionTemplateNumber.value in tables.AEROSOL_PDTNS is_aero_param = (self.parameterCategory in {13, 20}) and (self.parameterNumber in tables.AEROSOL_PARAMS) # Check table 4.205 aerosol presence is_aero_type = self.parameterCategory == 205 and self.parameterNumber == 1 return is_aero_template or is_aero_param or is_aero_type @property def _isChemical(self): """Check if GRIB2 message contains chemical data""" is_chem_template = self.productDefinitionTemplateNumber.value in tables.CHEMICAL_PDTNS is_chem_param = self.parameterCategory == 20 return is_chem_template or is_chem_param @property def gdtn(self): """Return Grid Definition Template Number""" return self.section3[4] @property def gdt(self): """Return Grid Definition Template.""" return self.gridDefinitionTemplate @property def pdtn(self): """Return Product Definition Template Number.""" return self.section4[1] @property def pdt(self): """Return Product Definition Template.""" return self.productDefinitionTemplate @property def drtn(self): """Return Data Representation Template Number.""" return self.section5[1] @property def drt(self): """Return Data Representation Template.""" return self.dataRepresentationTemplate @property def pdy(self): """Return the PDY ('YYYYMMDD').""" return "".join([str(i) for i in self.section1[5:8]]) @property def griddef(self): ``` -------------------------------- ### Station Point Interpolation Setup Source: https://noaa-mdl.github.io/grib2io/grib2io/xarray_backend.html Initializes station point coordinates for spatial interpolation. ```python da = self._obj # TODO ensure that y, x are rightmost dims; they should be if opening # with grib2io engine calls = np.asarray(calls) lats = np.asarray(lats) lons = np.asarray(lons) latitude = xr.DataArray(lats, dims=["station"]) longitude = xr.DataArray(lons, dims=["station"]) # create new coords ``` -------------------------------- ### Define Product Definition Template Classes Source: https://noaa-mdl.github.io/grib2io/grib2io/templates.html Examples of defining GRIB2 product definition templates using dataclasses, including surface attributes and specific template fields. ```python 2917 backgroundGeneratingProcessIdentifier: int = field(init=False, repr=False, default=BackgroundGeneratingProcessIdentifier()) 2918 hoursAfterDataCutoff: int = field(init=False, repr=False, default=HoursAfterDataCutoff()) 2919 minutesAfterDataCutoff: int = field(init=False, repr=False, default=MinutesAfterDataCutoff()) 2920 unitOfForecastTime: Grib2Metadata = field(init=False, repr=False, default=UnitOfForecastTime()) 2921 valueOfForecastTime: int = field(init=False, repr=False, default=ValueOfForecastTime()) 2922 2923 @classmethod 2924 def _attrs(cls): 2925 return [key for key in cls.__dataclass_fields__.keys() if not key.startswith("_")] ``` ```python @dataclass(init=False) class ProductDefinitionTemplateSurface: """Surface attributes for Product Definition Templates""" _fixedsfc1info: list = field(init=False, repr=False, default=FixedSfc1Info()) _fixedsfc2info: list = field(init=False, repr=False, default=FixedSfc2Info()) typeOfFirstFixedSurface: Grib2Metadata = field(init=False, repr=False, default=TypeOfFirstFixedSurface()) scaleFactorOfFirstFixedSurface: int = field(init=False, repr=False, default=ScaleFactorOfFirstFixedSurface()) scaledValueOfFirstFixedSurface: int = field(init=False, repr=False, default=ScaledValueOfFirstFixedSurface()) typeOfSecondFixedSurface: Grib2Metadata = field(init=False, repr=False, default=TypeOfSecondFixedSurface()) scaleFactorOfSecondFixedSurface: int = field(init=False, repr=False, default=ScaleFactorOfSecondFixedSurface()) scaledValueOfSecondFixedSurface: int = field(init=False, repr=False, default=ScaledValueOfSecondFixedSurface()) unitOfFirstFixedSurface: str = field(init=False, repr=False, default=UnitOfFirstFixedSurface()) valueOfFirstFixedSurface: int = field(init=False, repr=False, default=ValueOfFirstFixedSurface()) unitOfSecondFixedSurface: str = field(init=False, repr=False, default=UnitOfSecondFixedSurface()) valueOfSecondFixedSurface: int = field(init=False, repr=False, default=ValueOfSecondFixedSurface()) @classmethod def _attrs(cls): return [key for key in cls.__dataclass_fields__.keys() if not key.startswith("_")] ``` ```python @dataclass(init=False) class ProductDefinitionTemplate0(ProductDefinitionTemplateBase, ProductDefinitionTemplateSurface): """[Product Definition Template 0](https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_temp4-0.shtml)""" _len = 15 _num = 0 @classmethod def _attrs(cls): return [key for key in cls.__dataclass_fields__.keys() if not key.startswith("_")] ``` ```python @dataclass(init=False) class ProductDefinitionTemplate1(ProductDefinitionTemplateBase, ProductDefinitionTemplateSurface): """[Product Definition Template 1](https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_temp4-1.shtml)""" _len = 18 _num = 1 typeOfEnsembleForecast: Grib2Metadata = field(init=False, repr=False, default=TypeOfEnsembleForecast()) perturbationNumber: int = field(init=False, repr=False, default=PerturbationNumber()) numberOfEnsembleForecasts: int = field(init=False, repr=False, default=NumberOfEnsembleForecasts()) @classmethod def _attrs(cls): return [key for key in cls.__dataclass_fields__.keys() if not key.startswith("_")] ``` ```python @dataclass(init=False) class ProductDefinitionTemplate2(ProductDefinitionTemplateBase, ProductDefinitionTemplateSurface): """[Product Definition Template 2](https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_temp4-2.shtml)""" _len = 17 _num = 2 typeOfDerivedForecast: Grib2Metadata = field(init=False, repr=False, default=TypeOfDerivedForecast()) numberOfEnsembleForecasts: int = field(init=False, repr=False, default=NumberOfEnsembleForecasts()) @classmethod def _attrs(cls): return [key for key in cls.__dataclass_fields__.keys() if not key.startswith("_")] ``` ```python @dataclass(init=False) class ProductDefinitionTemplate5(ProductDefinitionTemplateBase, ProductDefinitionTemplateSurface): """[Product Definition Template 5](https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_temp4-5.shtml)""" _len = 22 _num = 5 forecastProbabilityNumber: int = field(init=False, repr=False, default=ForecastProbabilityNumber()) totalNumberOfForecastProbabilities: int = field(init=False, repr=False, default=TotalNumberOfForecastProbabilities()) typeOfProbability: Grib2Metadata = field(init=False, repr=False, default=TypeOfProbability()) scaleFactorOfThresholdLowerLimit: float = field(init=False, repr=False, default=ScaleFactorOfThresholdLowerLimit()) ``` -------------------------------- ### show_config() Source: https://noaa-mdl.github.io/grib2io Prints the build configuration information for the grib2io library, including library versions and compression support. ```APIDOC ## show_config() ### Description Prints grib2io build configuration information, including NCEPLIBS-g2c and NCEPLIBS-ip library versions, compression support (JPEG, PNG, AEC), and OpenMP status. ### Usage `grib2io.show_config()` ``` -------------------------------- ### Parse Data Model Example Source: https://noaa-mdl.github.io/grib2io/grib2io/xarray_backend.html Example usage of the parse_data_model function to convert a dataset to the 'nws-viz' model. ```python >>> ds2 = parse_data_model(ds, "nws-viz") >>> list(ds2.coords) ['forecast_reference_time', 'lead_time', 'time', 'percentile', ...] ``` -------------------------------- ### IcechunkWriter.__init__ Source: https://noaa-mdl.github.io/grib2io/grib2io/icechunk.html Initializes a new IcechunkWriter instance to manage GRIB2 reference manifests in an Icechunk virtual store. ```APIDOC ## IcechunkWriter(store_path: str, storage_config: Optional[Any] = None) ### Description Initializes the IcechunkWriter to write grib2io reference manifests into an Icechunk virtual store. ### Parameters - **store_path** (str) - Required - Path or URI for the Icechunk store (e.g., local directory or s3://bucket/prefix). - **storage_config** (Optional[Any]) - Optional - An Icechunk Storage object. Defaults to None, which creates a local filesystem storage. ``` -------------------------------- ### Lazy Import Guards for Optional Dependencies Source: https://noaa-mdl.github.io/grib2io/grib2io/xarray_backend.html Functions to ensure required optional packages are installed before use, raising an ImportError with installation instructions if missing. ```python def _ensure_kerchunk(): """Raise ``ImportError`` if *kerchunk* / *fsspec* reference support is not available.""" try: import fsspec # noqa: F401 except ImportError: raise ImportError("kerchunk is required for reference generation. Install with: pip install grib2io[kerchunk]") def _ensure_icechunk(): """Raise ``ImportError`` if *icechunk* is not available.""" try: import icechunk # noqa: F401 except ImportError: raise ImportError("icechunk is required for virtual store support. Install with: pip install grib2io[icechunk]") ``` -------------------------------- ### Package Initialization and Configuration Source: https://noaa-mdl.github.io/grib2io This module handles lazy imports for optional dependencies and provides the show_config function to display build-time configuration details. ```python 1from ._grib2io import ( 2 open, 3 interpolate, 4 interpolate_to_stations, 5 Grib2Message, 6 _Grib2Message, 7 Grib2GridDef, 8 msgs_from_index, 9 __doc__, 10) 11from . import tables, templates, utils 12 13try: 14 from . import __config__ 15 16 __version__ = __config__.grib2io_version 17 has_interpolation = __config__.has_interpolation 18 has_openmp_support = __config__.has_openmp_support 19 g2c_static = __config__.g2c_static 20 ip_static = __config__.ip_static 21 extra_objects = __config__.extra_objects 22except ImportError: 23 pass 24 25from .g2clib import __version__ as __g2clib_version__ 26from .g2clib import _has_jpeg 27from .g2clib import _has_png 28from .g2clib import _has_aec 29 30from .tables.originating_centers import _ncep_grib2_table_version 31 32__all__ = [ 33 "open", 34 "show_config", 35 "interpolate", 36 "interpolate_to_stations", 37 "tables", 38 "templates", 39 "utils", 40 "codecs", 41 "kerchunk", 42 "icechunk", 43 "Grib2Message", 44 "_Grib2Message", 45 "Grib2GridDef", 46 "msgs_from_index", 47 "__doc__", 48] 49 50has_jpeg_support = bool(_has_jpeg) 51has_png_support = bool(_has_png) 52has_aec_support = bool(_has_aec) 53 54ncep_grib2_table_version = _ncep_grib2_table_version 55g2c_version = __g2clib_version__ 56 57# --------------------------------------------------------------------------- 58# Lazy imports for optional-dependency modules 59# --------------------------------------------------------------------------- 60# These modules are only imported when explicitly accessed (e.g., 61# ``grib2io.codecs``, ``grib2io.kerchunk``, ``grib2io.icechunk``). 62# This keeps the core package lightweight and avoids ImportError when 63# optional dependencies (numcodecs, kerchunk, icechunk) are not installed. 64 65_LAZY_MODULES = {"codecs", "kerchunk", "icechunk"} 66 67 68def __getattr__(name: str): 69 if name in _LAZY_MODULES: 70 import importlib 71 72 module = importlib.import_module(f".{name}", __name__) 73 globals()[name] = module 74 return module 75 raise AttributeError(f"module {__name__!r} has no attribute {name!r}") 76 77 78def show_config(): 79 """Print grib2io build configuration information.""" 80 print(f"grib2io version {__version__} Configuration:") 81 print("") 82 print(f"NCEPLIBS-g2c library version: {__g2clib_version__}") 83 print(f"\tStatic library: {g2c_static}") 84 print(f"\tJPEG compression support: {has_jpeg_support}") 85 print(f"\tPNG compression support: {has_png_support}") 86 print(f"\tAEC compression support: {has_aec_support}") 87 print("") 88 print(f"NCEPLIPS-ip support: {has_interpolation}") 89 print(f"\tStatic library: {ip_static}") 90 print(f"\tOpenMP support: {has_openmp_support}") 91 print("") 92 print("Static libs:") 93 for lib in extra_objects: 94 print(f"\t{lib}") 95 print("") 96 print(f"NCEP GRIB2 Table Version: {_ncep_grib2_table_version}") ``` -------------------------------- ### Get Grid Y-Dimension Source: https://noaa-mdl.github.io/grib2io Property to retrieve the number of grid points in the y-direction. ```python 2491 @property 2492 def ny(self): 2493 """Number of grid points in y-direction.""" 2494 return int(self.gdt[8]) ``` -------------------------------- ### Get Grid X-Dimension Source: https://noaa-mdl.github.io/grib2io Property to retrieve the number of grid points in the x-direction. ```python 2486 @property 2487 def nx(self): 2488 """Number of grid points in x-direction.""" 2489 return int(self.gdt[7]) ``` -------------------------------- ### IcechunkWriter.__init__ Source: https://noaa-mdl.github.io/grib2io/grib2io/icechunk.html Initializes a new instance of the IcechunkWriter class to manage writing grib2io reference manifests into an Icechunk virtual store. ```APIDOC ## IcechunkWriter.__init__ ### Description Initializes the IcechunkWriter with a target store path and optional storage configuration. ### Parameters - **store_path** (str) - Required - Path or URI for the Icechunk store (e.g., local directory or s3:// URI). - **storage_config** (Optional[Any]) - Optional - An Icechunk Storage object. Defaults to None, which creates a local filesystem storage at store_path. ``` -------------------------------- ### Generate a Kerchunk reference manifest Source: https://noaa-mdl.github.io/grib2io/grib2io/kerchunk.html Demonstrates the basic workflow for initializing the generator, creating the manifest, and saving it to a JSON file. ```python >>> from grib2io.kerchunk import ReferenceGenerator >>> gen = ReferenceGenerator("gfs.grib2") >>> manifest = gen.generate() >>> gen.to_json("gfs_refs.json") ``` -------------------------------- ### Get Codec Configuration Source: https://noaa-mdl.github.io/grib2io/grib2io/codecs.html Returns a JSON-serializable dictionary containing all codec parameters. ```python 136 def get_config(self): 137 """Return a JSON-serializable dict of all codec parameters.""" 138 return { 139 "id": self.codec_id, 140 "drtn": self.drtn, 141 "drt": self.drt, 142 "gdtn": self.gdtn, 143 "gdt": self.gdt, 144 "gds": self.gds, 145 "nx": self.nx, 146 "ny": self.ny, 147 "bitmap_flag": self.bitmap_flag, 148 "bitmap_offset": self.bitmap_offset, 149 "bitmap_length": self.bitmap_length, 150 "scan_mode_flags": self.scan_mode_flags, 151 "type_of_values": self.type_of_values, 152 "number_of_data_points": self.number_of_data_points, 153 "number_of_packed_values": self.number_of_packed_values, 154 } ``` -------------------------------- ### Get Grid Shape Source: https://noaa-mdl.github.io/grib2io Property to retrieve the shape of the grid as a tuple (ny, nx). ```python 2501 @property 2502 def shape(self): 2503 """Shape of the grid.""" 2504 return (int(self.ny), int(self.nx)) ``` -------------------------------- ### grib2io.show_config Source: https://noaa-mdl.github.io/grib2io Prints the build configuration information for the grib2io package, including library versions and support for compression and interpolation. ```APIDOC ## grib2io.show_config() ### Description Displays the current build configuration, including NCEPLIBS-g2c version, compression support (JPEG, PNG, AEC), and NCEPLIBS-ip interpolation support status. ``` -------------------------------- ### Get Total Grid Points Source: https://noaa-mdl.github.io/grib2io Property to calculate the total number of grid points. ```python 2496 @property 2497 def npoints(self): 2498 """Total number of grid points.""" 2499 return int(self.gdt[7] * self.gdt[8]) ``` -------------------------------- ### GRIB2 File Initialization and Indexing Source: https://noaa-mdl.github.io/grib2io Handles file opening, gzip detection, and index generation or loading for GRIB2 files. ```python 257 self.messages = len(self._msgs) 258 259 # Get size for s3fs.core.S3File object. 260 self.size = self._filehandle.info()["size"] or 0 261 262 else: 263 self.current_message = 0 264 if "r" in mode: 265 self._filehandle = builtins.open(filename, mode=mode) 266 # Some GRIB2 files are gzipped, so check for that here, but 267 # raise error when using xarray backend. 268 # Gzip files contain a 2-byte header b'\x1f\x8b'. 269 if self._filehandle.read(2) == _GZIP_HEADER: 270 self._filehandle.close() 271 if _xarray_backend: 272 raise RuntimeError("Gzip GRIB2 files are not supported by the Xarray backend.") 273 self._filehandle = gzip.open(filename, mode=mode) 274 else: 275 self._filehandle = builtins.open(filename, mode=mode) 276 else: 277 self._filehandle = builtins.open(filename, mode=mode) 278 self.name = os.path.abspath(filename) 279 self.name = os.path.abspath(filename) 280 fstat = os.stat(self.name) 281 self.size = fstat.st_size 282 self._fileid = hashlib.sha1((self.name + str(fstat.st_ino) + str(self.size)).encode("ASCII")).hexdigest() 283 284 if "r" in self.mode: 285 self.indexfile = f"{self.name}_{self._fileid}.grib2ioidx" 286 if self.use_index and os.path.exists(self.indexfile): 287 try: 288 with builtins.open(self.indexfile, "rb") as file: 289 index = pickle.load(file) 290 self._index = index 291 except Exception as e: 292 warnings.warn( 293 f"found indexfile: {self.indexfile}, but unable to load it: {e}\n" 294 f"re-forming index from grib2file, but not writing indexfile" 295 ) 296 self._index = build_index(self._filehandle) 297 self._hasindex = True 298 else: 299 self._index = build_index(self._filehandle) 300 if self.save_index: 301 try: 302 serialize_index(self._index, self.indexfile) 303 except Exception as e: 304 warnings.warn(f"index was not serialized for future use: {e}") 305 306 self._msgs = msgs_from_index(self._index, filehandle=self._filehandle) 307 308 self.messages = len(self._msgs) 309 elif "w" or "x" in self.mode: 310 self.messages = 0 311 self.current_message = None 312 313 self.closed = self._filehandle.closed ``` -------------------------------- ### Get Levels by Variable Source: https://noaa-mdl.github.io/grib2io Returns a list of level strings for a given variable shortName. ```python def levels_by_var(self, name: str): """ Return a list of level strings given a variable shortName. """ ``` -------------------------------- ### Grib2Codec.__init__ Source: https://noaa-mdl.github.io/grib2io/grib2io/codecs.html Initializes the Grib2Codec with necessary GRIB2 metadata parameters required for unpacking data. ```APIDOC ## Grib2Codec.__init__(drtn, drt, gdtn, gdt, gds, nx, ny, bitmap_flag, bitmap_offset=None, bitmap_length=None, scan_mode_flags=None, type_of_values=0, number_of_data_points=0, number_of_packed_values=0) ### Description Constructs a new Grib2Codec instance. All parameters are JSON-serializable integers or lists extracted from GRIB2 message metadata. ### Parameters - **drtn** (int) - Data representation template number. - **drt** (List[int]) - Data representation template values. - **gdtn** (int) - Grid definition template number. - **gdt** (List[int]) - Grid definition template values. - **gds** (List[int]) - Grid definition section values. - **nx** (int) - Number of points along the x-axis. - **ny** (int) - Number of points along the y-axis. - **bitmap_flag** (int) - Flag indicating presence of a bitmap. - **bitmap_offset** (int, optional) - Offset for the bitmap. - **bitmap_length** (int, optional) - Length of the bitmap. - **scan_mode_flags** (List[int], optional) - Scan mode flags. - **type_of_values** (int) - Type of values (default 0). - **number_of_data_points** (int) - Total number of data points (default 0). - **number_of_packed_values** (int) - Total number of packed values (default 0). ``` -------------------------------- ### Grib2Codec.__init__ Source: https://noaa-mdl.github.io/grib2io/grib2io/codecs.html Initializes the Grib2Codec with necessary GRIB2 metadata parameters required for unpacking data. ```APIDOC ## Grib2Codec(drtn, drt, gdtn, gdt, gds, nx, ny, bitmap_flag, ...) ### Description Initializes a new instance of the Grib2Codec. This codec stores the metadata required to unpack GRIB2 section 7 bytes. ### Parameters - **drtn** (int) - Data Representation Template Number. - **drt** (List[int]) - Data Representation Template values. - **gdtn** (int) - Grid Definition Template Number. - **gdt** (List[int]) - Grid Definition Template values. - **gds** (List[int]) - Grid Definition Section values. - **nx** (int) - Number of points along the x-axis. - **ny** (int) - Number of points along the y-axis. - **bitmap_flag** (int) - Flag indicating presence of a bitmap. - **bitmap_offset** (Optional[int]) - Offset for the bitmap. - **bitmap_length** (Optional[int]) - Length of the bitmap. - **scan_mode_flags** (Optional[List[int]]) - Scan mode flags. - **type_of_values** (int) - Type of values (default 0). - **number_of_data_points** (int) - Total number of data points (default 0). - **number_of_packed_values** (int) - Total number of packed values (default 0). ``` -------------------------------- ### Get variables by level Source: https://noaa-mdl.github.io/grib2io Returns a list of unique variable shortName strings associated with a specific level. ```python 512 def vars_by_level(self, level: str): 513 """ 514 Return a list of variable shortName strings given a level. 515 516 Parameters 517 ---------- 518 level 519 Grib2Message variable level 520 521 Returns 522 ------- 523 vars_by_level 524 A list of unique variable shortName strings. 525 """ 526 return list(sorted(set([msg.shortName for msg in self.select(level=level)]))) ``` -------------------------------- ### Serialize manifest to Parquet Source: https://noaa-mdl.github.io/grib2io/grib2io/kerchunk.html Serializes the current manifest to a Parquet reference store. Requires the kerchunk library to be installed. ```python 172 def to_parquet(self, output_path: str) -> None: 173 """Serialize the manifest to a Parquet reference store. 174 175 Parameters 176 ---------- 177 output_path : str 178 Path to the output Parquet directory. 179 """ 180 _ensure_kerchunk() 181 if self._manifest is None: 182 self.generate() 183 from kerchunk.df import refs_to_dataframe 184 185 refs_to_dataframe(self._manifest, output_path) ``` -------------------------------- ### Display grib2io Configuration Source: https://noaa-mdl.github.io/grib2io Prints the current build configuration, including library versions and compression support. ```python 79def show_config(): 80 """Print grib2io build configuration information.""" 81 print(f"grib2io version {__version__} Configuration:") 82 print("") 83 print(f"NCEPLIBS-g2c library version: {__g2clib_version__}") 84 print(f"\tStatic library: {g2c_static}") 85 print(f"\tJPEG compression support: {has_jpeg_support}") 86 print(f"\tPNG compression support: {has_png_support}") 87 print(f"\tAEC compression support: {has_aec_support}") 88 print("") 89 print(f"NCEPLIPS-ip support: {has_interpolation}") 90 print(f"\tStatic library: {ip_static}") 91 print(f"\tOpenMP support: {has_openmp_support}") 92 print("") 93 print("Static libs:") 94 for lib in extra_objects: 95 print(f"\t{lib}") 96 print("") 97 print(f"NCEP GRIB2 Table Version: {_ncep_grib2_table_version}") ``` -------------------------------- ### to_parquet(output_path: str) -> None Source: https://noaa-mdl.github.io/grib2io/grib2io/kerchunk.html Serializes the manifest to a Parquet reference store. ```APIDOC ## to_parquet(output_path: str) -> None ### Description Serialize the manifest to a Parquet reference store. ### Parameters - **output_path** (str) - Required - Path to the output Parquet directory. ``` -------------------------------- ### Get levels by variable name Source: https://noaa-mdl.github.io/grib2io Returns a list of unique level strings associated with a specific variable shortName. ```python 496 def levels_by_var(self, name: str): 497 """ 498 Return a list of level strings given a variable shortName. 499 500 Parameters 501 ---------- 502 name 503 Grib2Message variable shortName 504 505 Returns 506 ------- 507 levels_by_var 508 A list of unique level strings. 509 """ 510 return list(sorted(set([msg.level for msg in self.select(shortName=name)]))) ``` -------------------------------- ### Get current GRIB2 message position Source: https://noaa-mdl.github.io/grib2io Returns the current file pointer position in units of GRIB2 messages. ```python 441 def tell(self): 442 """Returns the position of the file in units of GRIB2 Messages.""" 443 return self.current_message ``` -------------------------------- ### Configure Icechunk Storage Backends Source: https://noaa-mdl.github.io/grib2io/grib2io/icechunk.html Determines the appropriate Icechunk storage backend based on the provided URI prefix. ```python local_path = prefix[7:] # Remove "file://" # Remove trailing slash for the store path if local_path.endswith("/"): local_path = local_path[:-1] return icechunk.storage.local_filesystem_store(local_path) elif prefix.startswith("s3://"): return icechunk.storage.s3_store(region="us-east-1") elif prefix.startswith("gcs://"): return icechunk.storage.gcs_store(opts={}) elif prefix.startswith("http://") or prefix.startswith("https://"): return icechunk.storage.http_store(opts={}) else: # Default to local filesystem return icechunk.storage.local_filesystem_store(prefix) ``` -------------------------------- ### open(filename, mode='r', *, save_index=True, use_index=True, **kwargs) Source: https://noaa-mdl.github.io/grib2io Initializes a GRIB2 file object instance. This function opens a GRIB2 file for reading, writing, or creating, and handles indexing of GRIB2 messages. ```APIDOC ## open(filename, mode='r', *, save_index=True, use_index=True, **kwargs) ### Description Initializes a GRIB2 file object instance. This function opens a GRIB2 file for reading, writing, or creating, and handles indexing of GRIB2 messages. ### Parameters - **filename** (Union[bytes, str, Path, IO]) - Required - File path containing GRIB2 messages, bytes, or a file-like object. - **mode** (Literal['r', 'w', 'x']) - Optional - File access mode: 'r' for reading, 'w' for overwriting, 'x' for writing to a new file. Defaults to 'r'. - **save_index** (bool) - Optional - Whether to save a pickle-based index file. Defaults to True. - **use_index** (bool) - Optional - Whether to use an existing pickle-based index file. Defaults to True. ``` -------------------------------- ### Initialize GRIB2 File Object Source: https://noaa-mdl.github.io/grib2io Handles file-like objects, standard file paths, and index management during GRIB2 file initialization. ```python 239 self._msgs = msgs_from_index(self._index, filehandle=self._filehandle) 240 self.messages = len(self._msgs) 241 242 elif all(hasattr(filename, attr) for attr in ("read", "seek", "tell")): 243 # Handle file-like objects (including S3File, etc.) 244 self.current_message = 0 245 self._filehandle = filename 246 self.name = getattr(filename, "name", filename.__repr__()) 247 # For file-like objects, we can't get file stats, so create a unique ID 248 self._fileid = hashlib.sha1((self.name + str(id(filename))).encode("ASCII")).hexdigest() 249 # Disable index file usage for file-like objects since we can't save/load them 250 self.use_index = False 251 self.save_index = False 252 253 if "r" in self.mode: 254 # Build index from the file-like object 255 self._index = build_index(self._filehandle) 256 self._msgs = msgs_from_index(self._index, filehandle=self._filehandle) 257 self.messages = len(self._msgs) 258 259 # Get size for s3fs.core.S3File object. 260 self.size = self._filehandle.info()["size"] or 0 261 262 else: 263 self.current_message = 0 264 if "r" in mode: 265 self._filehandle = builtins.open(filename, mode=mode) 266 # Some GRIB2 files are gzipped, so check for that here, but 267 # raise error when using xarray backend. 268 # Gzip files contain a 2-byte header b'\x1f\x8b'. 269 if self._filehandle.read(2) == _GZIP_HEADER: 270 self._filehandle.close() 271 if _xarray_backend: 272 raise RuntimeError("Gzip GRIB2 files are not supported by the Xarray backend.") 273 self._filehandle = gzip.open(filename, mode=mode) 274 else: 275 self._filehandle = builtins.open(filename, mode=mode) 276 else: 277 self._filehandle = builtins.open(filename, mode=mode) 278 self.name = os.path.abspath(filename) 279 self.name = os.path.abspath(filename) 280 fstat = os.stat(self.name) 281 self.size = fstat.st_size 282 self._fileid = hashlib.sha1((self.name + str(fstat.st_ino) + str(self.size)).encode("ASCII")).hexdigest() 283 284 if "r" in self.mode: 285 self.indexfile = f"{self.name}_{self._fileid}.grib2ioidx" 286 if self.use_index and os.path.exists(self.indexfile): 287 try: 288 with builtins.open(self.indexfile, "rb") as file: 289 index = pickle.load(file) 290 self._index = index 291 except Exception as e: 292 warnings.warn( 293 f"found indexfile: {self.indexfile}, but unable to load it: {e}\n" 294 f"re-forming index from grib2file, but not writing indexfile" 295 ) 296 self._index = build_index(self._filehandle) 297 self._hasindex = True 298 else: 299 self._index = build_index(self._filehandle) 300 if self.save_index: 301 try: 302 serialize_index(self._index, self.indexfile) 303 except Exception as e: 304 warnings.warn(f"index was not serialized for future use: {e}") 305 306 self._msgs = msgs_from_index(self._index, filehandle=self._filehandle) 307 308 self.messages = len(self._msgs) 309 elif "w" or "x" in self.mode: 310 self.messages = 0 311 self.current_message = None 312 313 self.closed = self._filehandle.closed ``` -------------------------------- ### Open GRIB2 as Xarray DataTree Source: https://noaa-mdl.github.io/grib2io/grib2io/xarray_backend.html Opens a GRIB2 file as an xarray DataTree, requiring the DataTree functionality to be available in the installed xarray version. ```python def open_datatree( self, filename_or_obj, drop_variables=None, save_index=True, filters=None, stack_vertical=False, chunks=None, ) -> typing.Any: """ Open a GRIB2 file as an xarray DataTree. Parameters ---------- filename : str Path to the GRIB2 file. drop_variables : list, optional List of variables to exclude. filters : dict, optional Filter criteria for GRIB2 messages. stack_vertical : bool, optional If True, organize the tree with vertical layers stacked in a single dataset. chunks : int, dict or 'auto', optional If chunks is provided, it is used to load the dataset into a dask-backed dataset. Returns ------- xarray.DataTree A hierarchical DataTree representation of the GRIB2 data. """ if not _HAS_DATATREE: raise ImportError("xarray version does not support DataTree functionality.") if filters is None: filters = {} # Open the file without any filters first to get all messages with grib2io.open(filename_or_obj, save_index=save_index, _xarray_backend=True) as f: file_index = pd.DataFrame(f._index) file_index = file_index.assign(msg=list(f)) # Build tree structure from GRIB messages with specified options tree = build_datatree_from_grib( filename_or_obj, file_index, filters, stack_vertical=stack_vertical, drop_variables=drop_variables, chunks=chunks, ) # Update history for provenance now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") history = f"{now}: Initialized via grib2io.open_datatree\n" def _add_history(node): if node.ds is not None: node.ds.attrs["history"] = history + node.ds.attrs.get("history", "") for child in node.children.values(): _add_history(child) _add_history(tree) # Put warning here so it is the last message from likely other Xarray warnings. warnings.warn( "grib2io’s xarray backend DataTree support is experimental. The DataTree structure or attributes may change in future releases.", UserWarning, stacklevel=2, ) return tree ``` -------------------------------- ### Initialize Store with Zarr API Source: https://noaa-mdl.github.io/grib2io/grib2io/icechunk.html Handles the creation of the root group and parsing of manifest references for Zarr v3 compatibility. ```python def _write_create(self, refs: Dict[str, Any], store) -> None: """Write all refs to a fresh store using zarr's Python API. Creates the root group and all arrays via ``zarr.open_group`` / ``group.create_array`` so that Zarr v3 metadata is stored through the proper path that icechunk and zarr recognise. Coordinate (inline) data is written via the zarr Array write API, and virtual chunk refs for data variables are registered with ``store.set_virtual_ref``. Parameters ---------- refs : dict The ``"refs"`` dict from the manifest. store : IcechunkStore The Icechunk store to write to. """ import numpy as np import zarr # Collect all metadata by variable zarray_refs: Dict[str, str] = {} # var_name -> .zarray JSON str zattrs_refs: Dict[str, str] = {} # var_name -> .zattrs JSON str data_refs: Dict[str, Any] = {} # chunk key -> [uri, offset, len] inline_data_refs: Dict[str, str] = {} # chunk key -> inline value for key, value in refs.items(): if key == ".zgroup": pass # zarr.open_group handles the root group metadata elif key.endswith("/.zarray"): var_name = key.rsplit("/.zarray", 1)[0] zarray_refs[var_name] = value elif key.endswith("/.zattrs"): var_name = key.rsplit("/.zattrs", 1)[0] zattrs_refs[var_name] = value elif _is_virtual_ref(value): data_refs[key] = value elif _is_inline_data(value) and not _is_metadata_key(key): inline_data_refs[key] = value elif _is_metadata_key(key): pass # skip other metadata keys else: _logger.warning( "Skipping unrecognized manifest entry: %s = %r", key, value, ) # Create root group and all arrays via zarr's Python API so that # metadata is stored in the location that zarr/icechunk actually # reads, rather than as raw chunk bytes at the zarr.json key. root = zarr.open_group(store, mode="w", zarr_format=3) # Identify which variables have virtual (data) refs β€” those need # Grib2SerializerCodec as their zarr v3 serializer. data_var_names: set = {key.split("/")[0] for key in data_refs} # Import the zarr v3 serializer codec (registered at import time) ``` -------------------------------- ### Define Grib2Codec Class Source: https://noaa-mdl.github.io/grib2io/grib2io/codecs.html The Grib2Codec class implements the numcodecs.abc.Codec interface to decode GRIB2 section 7 bytes. It requires numcodecs to be installed. ```python 53class Grib2Codec(Codec): 54 """Zarr codec for decoding raw GRIB2 section 7 bytes. 55 56 All constructor parameters are JSON-serializable integers or lists 57 extracted from the GRIB2 message's section metadata at 58 reference-generation time. Together they carry everything needed to 59 call ``g2clib.unpack7()`` β€” the same low-level routine used by 60 ``grib2io._data()``. 61 """ 62 63 codec_id = "grib2io" 64 65 def __init__( 66 self, 67 drtn: int, 68 drt: List[int], 69 gdtn: int, 70 gdt: List[int], 71 gds: List[int], 72 nx: int, 73 ny: int, 74 bitmap_flag: int, 75 bitmap_offset: Optional[int] = None, 76 bitmap_length: Optional[int] = None, 77 scan_mode_flags: Optional[List[int]] = None, 78 type_of_values: int = 0, 79 number_of_data_points: int = 0, 80 number_of_packed_values: int = 0, 81 ): 82 if not _HAS_NUMCODECS: 83 raise ImportError("numcodecs is required for Grib2Codec. Install with: pip install grib2io[kerchunk]") 84 self.drtn = drtn 85 self.drt = list(drt) 86 self.gdtn = gdtn 87 self.gdt = list(gdt) 88 self.gds = list(gds) 89 self.nx = nx 90 self.ny = ny 91 self.bitmap_flag = bitmap_flag 92 self.bitmap_offset = bitmap_offset 93 self.bitmap_length = bitmap_length 94 self.scan_mode_flags = list(scan_mode_flags) if scan_mode_flags is not None else None 95 self.type_of_values = type_of_values 96 self.number_of_data_points = number_of_data_points 97 self.number_of_packed_values = number_of_packed_values 98 99 # ---- encode (not supported) ------------------------------------------ 100 101 def encode(self, buf): 102 """Not supported β€” GRIB2 encoding is handled by ``grib2io.open()``.""" 103 raise NotImplementedError("Grib2Codec is decode-only; GRIB2 encoding is handled by grib2io.open()") 104 105 # ---- decode ---------------------------------------------------------- 106 107 def decode(self, buf, out=None): 108 """Decode raw GRIB2 section 7 bytes into a NumPy array.""" 109 if not isinstance(buf, bytes): 110 buf = bytes(buf) 111 112 fld = _decode_grib2_bytes( 113 buf, 114 drtn=self.drtn, 115 drt=list(self.drt) if not isinstance(self.drt, list) else self.drt, 116 gdtn=self.gdtn, 117 gdt=list(self.gdt) if not isinstance(self.gdt, list) else self.gdt, 118 nx=self.nx, 119 ny=self.ny, 120 bitmap_flag=self.bitmap_flag, 121 bitmap_length=self.bitmap_length, 122 scan_mode_flags=list(self.scan_mode_flags) if self.scan_mode_flags is not None else None, 123 type_of_values=self.type_of_values, 124 number_of_data_points=self.number_of_data_points, 125 number_of_packed_values=self.number_of_packed_values, 126 ) 127 128 if out is not None: 129 np.copyto(out, fld) 130 return out 131 132 return fld 133 134 # ---- config serialization -------------------------------------------- 135 136 def get_config(self): 137 """Return a JSON-serializable dict of all codec parameters.""" 138 return { 139 "id": self.codec_id, 140 "drtn": self.drtn, 141 "drt": self.drt, 142 "gdtn": self.gdtn, 143 "gdt": self.gdt, 144 "gds": self.gds, 145 "nx": self.nx, 146 "ny": self.ny, 147 "bitmap_flag": self.bitmap_flag, 148 "bitmap_offset": self.bitmap_offset, 149 "bitmap_length": self.bitmap_length, 150 "scan_mode_flags": self.scan_mode_flags, 151 "type_of_values": self.type_of_values, 152 "number_of_data_points": self.number_of_data_points, 153 "number_of_packed_values": self.number_of_packed_values, 154 } 155 156 @classmethod 157 def from_config(cls, config): 158 """Reconstruct the codec from a configuration dict. 159 160 Parameters 161 ---------- 162 config : dict 163 Dictionary as returned by :meth:`get_config`. 164 165 Returns 166 ------- 167 Grib2Codec 168 """ 169 # Remove the 'id' key which is not a constructor parameter 170 cfg = {k: v for k, v in config.items() if k != "id"} 171 return cls(**cfg) ``` -------------------------------- ### GRIB2 File Initialization Method Source: https://noaa-mdl.github.io/grib2io Initializes the GRIB2 file object, supporting file paths, byte streams, and file-like objects with configurable indexing. ```python def __init__( self, filename: Union[bytes, str, Path, IO], mode: Literal["r", "w", "x"] = "r", *, save_index=True, use_index=True, _xarray_backend=False, **kwargs, ): ```