### msgspec Key-Value Client Implementation Source: https://jcristharif.com/msgspec/examples/asyncio-kv The `Client` class provides a convenient interface for interacting with the key-value server. It uses asyncio and msgspec to send requests (GET, PUT, DELETE, LISTKEYS) and receive responses. The `create` class method simplifies establishing a connection. ```python import asyncio import msgspec # Assume Request types and prefixed_recv/send are defined as in the Server code # For brevity, they are omitted here but would be needed for a complete client file. # Mock definitions for standalone client code if needed: class Get: def __init__(self, key: str): self.key = key class Put: def __init__(self, key: str, value: str): self.key = key self.value = value class Del: def __init__(self, key: str): self.key = key class ListKeys: pass async def prefixed_recv(reader: asyncio.StreamReader) -> bytes: # Mock implementation pass async def prefixed_send(writer: asyncio.StreamWriter, data: bytes) -> None: # Mock implementation pass class Client: """An example TCP key-value client using asyncio and msgspec.""" def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): self.reader = reader self.writer = writer @classmethod async def create(cls, host: str = "127.0.0.1", port: int = 8888): """Create a new client""" reader, writer = await asyncio.open_connection(host, port) return cls(reader, writer) async def close(self) -> None: """Close the client.""" self.writer.close() await self.writer.wait_closed() async def request(self, req): """Send a request and await the response""" # Encode and send the request buffer = msgspec.msgpack.encode(req) await prefixed_send(self.writer, buffer) # Receive and decode the response buffer = await prefixed_recv(self.reader) return msgspec.msgpack.decode(buffer) async def get(self, key: str) -> str | None: """Get a key from the KV store, returning None if not present""" return await self.request(Get(key)) async def put(self, key: str, val: str) -> None: """Put a key-val pair in the KV store""" return await self.request(Put(key, val)) async def delete(self, key: str) -> None: """Delete a key-val pair from the KV store""" return await self.request(Del(key)) async def list_keys(self) -> list[str]: """List all keys in the KV store""" return await self.request(ListKeys()) # Example usage snippet (requires running server): # async def main(): # client = await Client.create() # await client.put("foo", "bar") # await client.put("fizz", "buzz") # print(await client.get("foo")) # print(await client.list_keys()) # await client.delete("fizz") # print(await client.list_keys()) # await client.close() # # if __name__ == "__main__": # asyncio.run(main()) ``` -------------------------------- ### msgspec Key-Value Server Implementation Source: https://jcristharif.com/msgspec/examples/asyncio-kv The `Server` class implements a TCP key-value server using asyncio. It utilizes msgspec for encoding responses and decoding requests, ensuring type safety and performance. The server handles GET, PUT, DELETE, and LISTKEYS operations. ```python import asyncio import msgspec from typing import Any # Define request types class Get: def __init__(self, key: str): self.key = key class Put: def __init__(self, key: str, value: str): self.key = key self.value = value class Del: def __init__(self, key: str): self.key = key class ListKeys: pass Request = Get | Put | Del | ListKeys async def prefixed_recv(reader: asyncio.StreamReader) -> bytes: # Helper function to receive data with a prefix length # (Implementation assumed to be present or from a library) pass async def prefixed_send(writer: asyncio.StreamWriter, data: bytes) -> None: # Helper function to send data with a prefix length # (Implementation assumed to be present or from a library) pass class Server: """An example TCP key-value server using asyncio and msgspec""" def __init__(self, host: str = "127.0.0.1", port: int = 8888): self.host = host self.port = port self.kv: dict[str, str] = {} # A msgpack encoder for encoding responses self.encoder = msgspec.msgpack.Encoder() # A *typed* msgpack decoder for decoding requests. If a request doesn't # match the specified types, a nice error will be raised. self.decoder = msgspec.msgpack.Decoder(Request) async def handle_connection( self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter ): """Handle the full lifetime of a single connection""" print("Connection opened") while True: try: # Receive and decode a request buffer = await prefixed_recv(reader) req = self.decoder.decode(buffer) # Process the request resp = await self.handle_request(req) # Encode and write the response buffer = self.encoder.encode(resp) await prefixed_send(writer, buffer) except EOFError: print("Connection closed") return async def handle_request(self, req: Request) -> Any: """Handle a single request and return the result (if any)""" # We use pattern matching here to branch on the different message types. # You could just as well use an if-else statement, but pattern matching # works pretty well here. match req: case Get(key): # Return the value for a key, or None if missing return self.kv.get(key) case Put(key, val): # Add a new key-value pair self.kv[key] = val return None case Del(key): # Remove a key-value pair if it exists self.kv.pop(key, None) return None case ListKeys(): # Return a list of all keys in the store return sorted(self.kv) async def serve_forever(self) -> None: server = await asyncio.start_server( self.handle_connection, self.host, self.port ) print(f"Serving on tcp://{self.host}:{self.port}...") async with server: await server.serve_forever() def run(self) -> None: """Run the server until ctrl-C""" asyncio.run(self.serve_forever()) if __name__ == "__main__": Server().run() ``` -------------------------------- ### Instantiate and observe default values of Example struct in Python Source: https://jcristharif.com/msgspec/_sources/structs.rst This snippet shows the instantiation of the 'Example' struct with default values and demonstrates that mutable default collections like lists are unique per instance. ```python Example() Example(a=2) Example().c is Example().c ``` -------------------------------- ### Define Request Structs for Key-Value Server - Python Source: https://jcristharif.com/msgspec/examples/asyncio-kv Defines the request types for the asyncio TCP key-value server using msgspec.Struct. Each struct is tagged to be usable within a union of request types. Supported operations include Get, Put, Delete, and ListKeys. ```python # Define some request types. We set `tag=True` on each type so they can be used # in a "tagged-union" defining the request types. class Get(msgspec.Struct, tag=True): key: str class Put(msgspec.Struct, tag=True): key: str val: str class Del(msgspec.Struct, tag=True): key: str class ListKeys(msgspec.Struct, tag=True): pass ``` -------------------------------- ### Python msgspec Struct Pattern Matching Example Source: https://jcristharif.com/msgspec/structs Demonstrates the usage of msgspec.Struct types within pattern matching blocks, assuming Python 3.10 or later is used, as per PEP 636 examples. ```python # Example for Python 3.10+ pattern matching # Assuming a struct definition like: # class MyStruct(msgspec.Struct): # a: int # b: str # Usage in pattern matching would look like: # match instance_of_my_struct: # case MyStruct(a=1, b="hello"): # print("Matches") # case MyStruct(a=val, b=other_val): # print(f"Matched with a={val}, b={other_val}") ``` -------------------------------- ### Create and use User struct instances in Python Source: https://jcristharif.com/msgspec/_sources/structs.rst This example shows how to instantiate the 'User' struct defined previously. It illustrates creating instances with and without optional arguments, accessing fields, and comparing instances for equality. ```python alice = User("alice", groups={"admin", "engineering"}) alice bob = User("bob", email="bob@company.com") bob alice.name bob.groups alice == bob alice == User("alice", groups={"admin", "engineering"}) ``` -------------------------------- ### Python IPyhton3 GeoJSON Loading Example Source: https://jcristharif.com/msgspec/_sources/examples/geojson.rst This IPython3 code demonstrates loading GeoJSON data using the previously defined msgspec_geojson module. It reads data from a file and parses it into a high-level msgspec.Struct object, showcasing its ability to load and validate complex geographical data structures. ```ipython3 import msgspec_geojson with open("canada.json", "rb") as f: data = f.read() canada = msgspec_geojson.loads(data) type(canada) # loaded as high-level, validated object canada.features[0].properties ``` -------------------------------- ### Define a struct with static, dynamic, and mutable default values in Python Source: https://jcristharif.com/msgspec/_sources/structs.rst This example showcases how to define a struct 'Example' with different types of default values for its fields: a static integer default, a dynamic UUID generated by a factory, and an empty list as a default. ```python from msgspec import Struct, field import uuid class Example(Struct): a: int = 1 b: uuid.UUID = field(default_factory=uuid.uuid4) c: list[int] = [] ``` -------------------------------- ### msgspec Struct Pattern Matching Example Source: https://jcristharif.com/msgspec/structs Demonstrates pattern matching with msgspec Structs, allowing for conditional logic based on struct field values. Requires Python 3.10+. ```python import msgspec class Point(msgspec.Struct): x: float y: float def where_is(point): match point: case Point(0, 0): print("Origin") case Point(0, y): print(f"Y={y}") case Point(x, 0): print(f"X={x}") case Point(): print("Somewhere else") case _: print("Not a point") where_is(Point(0, 6)) ``` -------------------------------- ### Create a Struct with a Custom Metaclass (KwOnlyStructMeta) Source: https://jcristharif.com/msgspec/api Shows how to define a custom metaclass, KwOnlyStructMeta, that inherits from msgspec.StructMeta. This metaclass enforces keyword-only arguments for struct fields by default. A KwOnlyStruct base class is created using this metaclass, and an example demonstrates its usage. ```python >>> from msgspec import Struct, StructMeta >>> class KwOnlyStructMeta(StructMeta): ... def __new__(mcls, name, bases, namespace, **struct_config): ... struct_config.setdefault("kw_only", True) ... return super().__new__(mcls, name, bases, namespace, **struct_config) ... >>> class KwOnlyStruct(Struct, metaclass=KwOnlyStructMeta): ... ... >>> class Example(KwOnlyStruct): ... a: str = "" ... b: int ... >>> Example(b=123) Example(a='', b=123) ``` -------------------------------- ### Example Invalid TOML Content Source: https://jcristharif.com/msgspec/_sources/examples/pyproject-toml.rst This is an example of an invalid TOML content for a pyproject.toml file, specifically demonstrating an incorrect type for the 'build-system.requires' field. ```toml [build-system] requires = "hatchling" build-backend = "hatchling.build" [project] name = "myproject" version = "0.1.0" description = "a super great library" authors = [ {name = "alice shmalice", email = "alice@company.com"} ] ``` -------------------------------- ### Custom Struct Policies with Metaclasses Source: https://jcristharif.com/msgspec/_sources/structs.rst Shows how to define project-wide policies for msgspec.Struct subclasses by extending `msgspec.StructMeta`. The example demonstrates enforcing `kw_only=True` for all derived structs. ```python from msgspec import Struct, StructMeta class KwOnlyStructMeta(StructMeta): def __new__(mcls, name, bases, namespace, **struct_config): struct_config.setdefault("kw_only", True) return super().__new__(mcls, name, bases, namespace, **struct_config) class KwOnlyStruct(Struct, metaclass=KwOnlyStructMeta): ... class Example(KwOnlyStruct): a: str = "" b: int Example() Example(b=123) ``` -------------------------------- ### Use the to_dict method of the Point struct in Python Source: https://jcristharif.com/msgspec/_sources/structs.rst This example demonstrates calling the custom 'to_dict' method on an instance of the 'Point' struct, showing the conversion of struct data into a dictionary format. ```python p = Point(1.0, 2.0) p.to_dict() ``` -------------------------------- ### Load GeoJSON Data with msgspec Source: https://jcristharif.com/msgspec/examples/geojson Demonstrates loading GeoJSON data from a file using the previously defined msgspec decoder. The loaded data is validated and represented as high-level msgspec.Struct objects. ```python import msgspec_geojson with open("canada.json", "rb") as f: data = f.read() canada = msgspec_geojson.loads(data) print(type(canada)) print(canada.features[0].properties) ``` -------------------------------- ### Dynamically Define a User Struct with defstruct Source: https://jcristharif.com/msgspec/api Illustrates the dynamic creation of a 'User' Struct using msgspec.defstruct. This method allows defining structs programmatically with specified fields, types, and default values or factories. The example defines fields for name, email, and groups, with groups defaulting to an empty set. ```python >>> from msgspec import defstruct, field >>> User = defstruct( ... 'User', ... [ ... ('name', str), ... ('email', str | None, None), ... ('groups', set[str], field(default_factory=set)), ... ], ... ) >>> User('alice') User(name='alice', email=None, groups=set()) ``` -------------------------------- ### Demonstrate Schema Evolution with msgspec - Python Source: https://jcristharif.com/msgspec/schema-evolution Demonstrates msgspec's schema evolution by serializing a User2 object and deserializing it with an older User decoder, and vice versa. Shows how missing fields get default values and unknown fields are skipped. ```python >>> old_dec = msgspec.json.Decoder(User) >>> new_dec = msgspec.json.Decoder(User2) >>> new_msg = msgspec.json.encode( ... User2("bob", groups={"finance"}, phone="512-867-5309") ... ) >>> old_dec.decode(new_msg) # deserializing a new msg with an older decoder User(name='bob', groups={'finance'}, email=None) >>> old_msg = msgspec.json.encode( ... User("alice", groups={"admin", "engineering"}) ... ) >>> new_dec.decode(old_msg) # deserializing an old msg with a new decoder User2(name="alice", groups={"admin", "engineering"}, email=None, phone=None) ``` -------------------------------- ### Python msgspec Struct Field Ordering Example Source: https://jcristharif.com/msgspec/structs Demonstrates the default field ordering in msgspec.Struct definitions and the resulting `__init__` signature. Fields are ordered as defined, affecting positional vs. keyword arguments. ```python import msgspec class Example(msgspec.Struct): a: str b: int = 0 # The generated __init__() for User looks like: def __init__(self, a: str, b: int = 0): pass ``` -------------------------------- ### Get Struct Field Information with msgspec.structs.fields Source: https://jcristharif.com/msgspec/api Retrieves information about the fields of a msgspec.Struct type or instance. It returns a tuple of FieldInfo objects, detailing each field's name, encoding name, type, and default values. ```python import msgspec class User(msgspec.Struct): user_id: int name: str = "" fields_info = msgspec.structs.fields(User) for field in fields_info: print(f"Name: {field.name}, Encode Name: {field.encode_name}, Type: {field.type}") ``` -------------------------------- ### Encode and Decode Pydantic Dataclasses with msgspec Source: https://jcristharif.com/msgspec/_sources/supported-types.rst Demonstrates using Pydantic dataclasses with msgspec for JSON encoding and decoding. This example shows how Pydantic handles type coercion during initialization and how msgspec serializes the Pydantic dataclass instance. ```python >>> from datetime import datetime >>> from pydantic.dataclasses import dataclass >>> @dataclass ... class User: ... id: int ... name: str = 'John Doe' ... signup_ts: datetime | None = None >>> user = User(id='42', signup_ts='2032-06-21T12:00') >>> user User(id=42, name='John Doe', signup_ts=datetime.datetime(2032, 6, 21, 12, 0)) >>> msgspec.json.encode(user) b'{"id":42,"name":"John Doe","signup_ts":"2032-06-21T12:00:00"}' ``` -------------------------------- ### Encode/Decode Structs as Arrays with msgspec (Python) Source: https://jcristharif.com/msgspec/perf-tips This example shows how to define a struct with `array_like=True` and then encode and decode instances of this struct using MessagePack. This method omits field names, resulting in more compact and faster data transfer. The output is a JSON byte string representing an array. ```python >>> class Example(msgspec.Struct, array_like=True): ... my_first_field: str ... my_second_field: int >>> x = Example("some string", 2) >>> msg = msgspec.json.encode(x) >>> msg b'["some string",2]' >>> msgspec.json.decode(msg, type=Example) Example(my_first_field="some string", my_second_field=2) ``` -------------------------------- ### Add TOML Support with msgspec Converters Source: https://jcristharif.com/msgspec/_sources/converters.rst Illustrates how to add TOML serialization support by wrapping Python's `tomllib` module. This example uses `msgspec.convert` with specific `builtin_types` for datetime objects and sets `str_keys=True` as TOML only supports string keys. ```python import datetime import tomllib from typing import Any import msgspec def decode(msg, *, type=Any, dec_hook=None): return msgspec.convert( tomllib.loads(msg), type, builtin_types=(datetime.datetime, datetime.date, datetime.time), str_keys=True, dec_hook=dec_hook, ) # Example usage (assuming 'toml_data' is a string containing TOML content): # toml_data = """ # x = 1 # y = 2 # timestamp = 2023-10-27T10:00:00 # """ # decoded_data = decode(toml_data, type={'x': int, 'y': int, 'timestamp': datetime.datetime}) # print(decoded_data) ``` -------------------------------- ### Benchmark GeoJSON Parsing Performance Source: https://jcristharif.com/msgspec/examples/geojson Compares the performance of msgspec's GeoJSON parsing against other popular libraries like orjson, the standard library's json module, and the geojson library. This benchmark highlights msgspec's significant speed advantage. ```python # Assuming 'data' is loaded GeoJSON byte string from previous example # Benchmark msgspec # %timeit msgspec_geojson.loads(data) # Benchmark orjson # import orjson # %timeit orjson.loads(data) # Benchmark json # import json # %timeit json.loads(data) # Benchmark geojson # import geojson # %timeit geojson.loads(data) ``` -------------------------------- ### msgspec Strict vs. Lax Decoding Modes Source: https://jcristharif.com/msgspec/usage Explains the difference between msgspec's default 'strict' decoding mode, which disallows implicit type conversions, and 'lax' mode, which allows more flexible conversions. Examples show how strict mode raises errors for incompatible types, while lax mode attempts conversions. ```python >>> msgspec.json.decode(b'[1, 2, "3"]', type=list[int]) Traceback (most recent call last): File "", line 1, in msgspec.ValidationError: Expected `int`, got `str` - at `$[2]` ``` ```python >>> msgspec.json.decode(b'[1, 2, "3"]', type=list[int], strict=False) [1, 2, 3] ``` -------------------------------- ### Asyncio TCP Key-Value Server Utilities - Python Source: https://jcristharif.com/msgspec/examples/asyncio-kv Provides utility functions for sending and receiving length-prefixed messages over an asyncio stream. This framing mechanism helps in efficiently determining message boundaries before decoding. It uses big-endian integer representation for message lengths. ```python from __future__ import annotations import asyncio from typing import Any import msgspec # Some utilities for writing and reading length-prefix framed messages. Using # length-prefixed framing makes it easier for the reader to determine the # boundaries of each message before passing it to msgspec to be decoded. async def prefixed_send(stream: asyncio.StreamWriter, buffer: bytes) -> None: """Write a length-prefixed buffer to the stream""" # Encode the message length as a 4 byte big-endian integer. prefix = len(buffer).to_bytes(4, "big") # Write the prefix and buffer to the stream. stream.write(prefix) stream.write(buffer) await stream.drain() async def prefixed_recv(stream: asyncio.StreamReader) -> bytes: """Read a length-prefixed buffer from the stream""" # Read the next 4 byte prefix prefix = await stream.readexactly(4) # Convert the prefix back into an integer for the next message length n = int.from_bytes(prefix, "big") # Read in the full message buffer return await stream.readexactly(n) ``` -------------------------------- ### msgspec ValidationError Example Source: https://jcristharif.com/msgspec/_sources/usage.rst Shows an example of a `ValidationError` raised by msgspec when decoded data does not match the specified type, specifically when a list of strings is expected but an integer is found. ```python >>> msgspec.json.decode( ... b'{"name": "bill", "groups": ["devops", 123]}', ... type=User ... ) Traceback (most recent call last): File "", line 1, in msgspec.ValidationError: Expected `str`, got `int` - at `$.groups[1]` ``` -------------------------------- ### Asyncio TCP Key-Value Server and Client Implementation (Python) Source: https://jcristharif.com/msgspec/_sources/examples/asyncio-kv.rst Implements an asynchronous TCP server and client for a key-value store using msgspec for serialization. The server handles 'get', 'put', 'delete', and 'list_keys' operations. Messages are encoded using msgpack with length-prefix framing. ```python import asyncio import msgpack from typing import Literal, Union, cast from msgspec import Struct, msgpack_decode, msgpack_encode class Get(Struct, kw_only=True): key: str class Put(Struct, kw_only=True): key: str val: str class Delete(Struct, kw_only=True): key: str class ListKeys(Struct): pass Request = Union[Get, Put, Delete, ListKeys] Response = Union[str, None, list[str]] def main(): async def handle_connection(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): try: while True: length_bytes = await reader.readexactly(4) length = int.from_bytes(length_bytes, "big") payload = await reader.readexactly(length) request = msgpack_decode(payload, type=Request) response = await handle_request(request) response_encoded = msgpack_encode(response) writer.write(len(response_encoded).to_bytes(4, "big")) writer.write(response_encoded) await writer.drain() except asyncio.IncompleteReadError: print("Connection closed") finally: writer.close() await writer.wait_closed() async def handle_request(request: Request) -> Response: if isinstance(request, Get): return store.get(request.key) elif isinstance(request, Put): store[request.key] = request.val return None elif isinstance(request, Delete): store.pop(request.key, None) return None elif isinstance(request, ListKeys): return list(store.keys()) else: raise TypeError(f"Unknown request type: {type(request)}") store = {} asyncio.run(main_server()) async def main_server(): server = await asyncio.start_server(handle_connection, "127.0.0.1", 8888) addr = server.sockets.pop().getsockname() print(f"Serving on tcp://{addr[0]}:{addr[1]}...") async with server: await server.serve_forever() class Client: def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): self._reader = reader self._writer = writer async def _send(self, request: Request) -> Response: encoded = msgpack_encode(request) self._writer.write(len(encoded).to_bytes(4, "big")) self._writer.write(encoded) await self._writer.drain() length_bytes = await self._reader.readexactly(4) length = int.from_bytes(length_bytes, "big") payload = await self._reader.readexactly(length) return cast(Response, msgpack_decode(payload)) async def put(self, key: str, val: str) -> None: await self._send(Put(key=key, val=val)) async def get(self, key: str) -> str | None: return await self._send(Get(key=key)) async def delete(self, key: str) -> None: await self._send(Delete(key=key)) async def list_keys(self) -> list[str]: return await self._send(ListKeys()) @classmethod async def create(cls, host: str = "127.0.0.1", port: int = 8888) -> "Client": reader, writer = await asyncio.open_connection(host, port) return cls(reader, writer) def close(self): self._writer.close() async def wait_closed(self): await self._writer.wait_closed() if __name__ == "__main__": main() ``` -------------------------------- ### Build Performance Plot with Vega-Lite Source: https://jcristharif.com/msgspec/_sources/benchmarks.rst Generates a bar chart to visualize encoding and decoding times for different libraries and methods. It calculates time units (ns, μs, ms) based on the maximum observed time and formats tooltips for clarity. This function requires the `vegaEmbed` library. ```javascript function buildPlot(div, rows, title) { var max_time = 0; for (i = 0; i < rows.length; i++) { var total = rows[i].encode + rows[i].decode; if (total > max_time) { max_time = total; } } var time_unit, scale; if (max_time < 1e-6) { time_unit = "ns"; scale = 1e9; } else if (max_time < 1e-3) { time_unit = "μs"; scale = 1e6; } else { time_unit = "ms"; scale = 1e3; } var columns = ["encode", "decode", "total"]; var data = []; for (i = 0; i < rows.length; i++) { var label = rows[i].label; var et = rows[i].encode * scale; var dt = rows[i].decode * scale; var tt = et + dt; data.push({library: label, method: "encode", time: et}); data.push({library: label, method: "decode", time: dt}); data.push({library: label, method: "total", time: tt}); } var spec = { "$schema": "https://vega.github.io/schema/vega-lite/v5.2.0.json", "title": title, "config": { "view": {"stroke": null}, "legend": {"title": null, "labelFontSize": 12}, "title": {"fontSize": 14, "offset": 10}, "axis": {"titleFontSize": 12, "titlePadding": 10} }, "width": "container", "data": {"values": data}, "transform": [ { "calculate": "join([format(datum.time, '.3'), ' ${time_unit}'], '')", "as": "tooltip", } ], "mark": "bar", "encoding": { "color": { "field": "method", "type": "nominal", "scale": {"scheme": "tableau20"}, "sort": columns, }, "row": { "field": "library", "header": { "orient": "left", "labelAngle": 0, "labelAlign": "left", "labelFontSize": 12 }, "sort": {"field": "time", "op": "sum", "order": "ascending"}, "title": null, "type": "nominal", }, "tooltip": {"field": "tooltip", "type": "nominal"}, "x": { "axis": {"grid": false, "title": `Time (${time_unit})`}, "field": "time", "type": "quantitative", }, "y": { "axis": {"labels": false, "ticks": false, "title": null}, "field": "method", "type": "nominal", "sort": columns, }, }, }; vegaEmbed(div, spec); } ``` -------------------------------- ### Reuse JSON Encoder in Python Source: https://jcristharif.com/msgspec/_sources/perf-tips.rst To optimize JSON encoding performance, create a msgspec.json.Encoder instance once and reuse it for multiple encoding operations, avoiding repeated setup costs. ```python import msgspec encoder = msgspec.json.Encoder() # Create once for msg in msgs: data = encoder.encode(msg) # reuse multiple times ``` -------------------------------- ### Encode and Decode Pydantic Dataclasses with msgspec Source: https://jcristharif.com/msgspec/supported-types Demonstrates using msgspec to encode and decode Pydantic dataclasses. This example highlights type coercion for fields like datetime objects. ```python >>> from datetime import datetime >>> from pydantic.dataclasses import dataclass >>> @dataclass ... class User: ... id: int ... name: str = 'John Doe' ... signup_ts: datetime | None = None >>> user = User(id='42', signup_ts='2032-06-21T12:00') >>> user User(id=42, name='John Doe', signup_ts=datetime.datetime(2032, 6, 21, 12, 0)) >>> msgspec.json.encode(user) b'{"id":42,"name":"John Doe","signup_ts":"2032-06-21T12:00:00"}' ``` -------------------------------- ### Struct Definition and Configuration Source: https://jcristharif.com/msgspec/api This section details the definition and configuration of `msgspec.Struct` for creating efficient serializable objects. It covers field definitions using type annotations and various configuration options that can be passed to the class definition. ```APIDOC ## Structs ### `_msgspec.Struct` A base class for defining efficient serializable objects. Fields are defined using type annotations. Fields may optionally have default values, which result in keyword parameters to the constructor. Structs automatically define `__init__`, `__eq__`, `__repr__`, and `__copy__` methods. #### Configuration Options: * **frozen** (`bool`, default `False`): Whether instances are pseudo-immutable. If `True`, attribute assignment is disabled and a `__hash__` is defined. * **order** (`bool`, default `False`): If `True`, `__lt__`, `__le__`, `__gt__`, and `__ge__` methods will be generated. * **eq** (`bool`, default `True`): If `True`, an `__eq__` method will be generated. Set to `False` to compare based on instance identity alone. * **kw_only** (`bool`, default `False`): If `True`, all fields are treated as keyword-only arguments in the generated `__init__` method. * **omit_defaults** (`bool`, default `False`): Whether fields should be omitted from encoding if their value is the default. * **forbid_unknown_fields** (`bool`, default `False`): If `True`, an error is raised if an unknown field is encountered during decoding. If `False`, unknown fields are skipped. * **tag** (`str`, `int`, `bool`, `callable`, or `None`, default `None`): Used with `tag_field` for tagged union support. If non-`None`, the struct is considered "tagged". * **tag_field** (`str` or `None`, default `None`): The field name to use for tagged union support. Defaults to `"type"` if `tag` is non-`None`. * **rename** (`str`, `mapping`, `callable`, or `None`, default `None`): Controls renaming of field names during encoding/decoding. Can be presets like `"lower"`, `"upper"`, `"camel"`, `"pascal"`, `"kebab"`, a mapping, or a callable. * **repr_omit_defaults** (`bool`, default `False`): Whether fields should be omitted from the generated `repr` if their value is the default. * **array_like** (`bool`, default `False`): If `True`, the struct is treated as array-like during encoding/decoding instead of dict-like. * **gc** (`bool`, default `True`): Whether garbage collection is enabled for this type. Disabling may reduce GC pressure but requires careful handling of reference cycles. * **weakref** (`bool`, default `False`): Whether instances support weak references. * **dict** (`bool`, default `False`): Whether instances will include a `__dict__`. Setting to `True` allows adding undeclared attributes. * **cache_hash** (`bool`, default `False`): If enabled, the hash of a frozen struct instance is computed once and cached. #### Example: ```python class Dog(Struct): # Fields defined with type annotations name: str age: int breed: str = "Unknown" # Example of using configuration options class Config: frozen = True order = True rename = "camel" ``` ``` -------------------------------- ### Get Information about Multiple msgspec Types Source: https://jcristharif.com/msgspec/api The `multi_type_info` function efficiently retrieves detailed information for multiple msgspec-compatible types provided as an iterable. It returns a tuple of `Type` objects, one for each input type. ```python import msgspec # Example usage types_info = msgspec.inspect.multi_type_info([int, float, list[str]]) for type_info in types_info: print(type_info) ``` -------------------------------- ### Convert Struct to Tuple using msgspec.structs.astuple Source: https://jcristharif.com/msgspec/api Demonstrates how to convert a msgspec.Struct instance into a tuple. This is useful for serializing data in a compact, ordered format. ```python import msgspec class Point(msgspec.Struct): x: int y: int obj = Point(x=1, y=2) print(msgspec.structs.astuple(obj)) ``` -------------------------------- ### Constrain List Elements to Positive Integers Source: https://jcristharif.com/msgspec/_sources/constraints.rst Illustrates how to use `typing.Annotated` and `msgspec.Meta` to enforce constraints on the values within a decoded list. This example ensures all integers are greater than 0. ```python >>> from typing import Annotated >>> PositiveInt = Annotated[int, msgspec.Meta(gt=0)] >>> msgspec.json.decode(b'[1, 2, 3]', type=list[PositiveInt]) [1, 2, 3] >>> msgspec.json.decode(b'[1, 2, -1]', type=list[PositiveInt]) Traceback (most recent call last): File "", line 1, in msgspec.ValidationError: Expected `int` >= 1 - at `$[2]` ``` -------------------------------- ### Python IPyhton3 GeoJSON Performance Benchmarking Source: https://jcristharif.com/msgspec/_sources/examples/geojson.rst This IPython3 code benchmarks the performance of loading GeoJSON data using msgspec_geojson against other libraries like orjson, json, and geojson. It uses the %timeit magic command to measure the execution time of the loads function for each library, highlighting msgspec's speed advantage. ```ipython3 %timeit msgspec_geojson.loads(data) # benchmark msgspec %timeit orjson.loads(data) # benchmark orjson %timeit json.loads(data) # benchmark json %timeit geojson.loads(data) # benchmark geojson ``` -------------------------------- ### Load and Decode Starlette's pyproject.toml Source: https://jcristharif.com/msgspec/examples/pyproject-toml Demonstrates loading the pyproject.toml file for the Starlette project from a URL using `urllib.request` and then decoding its content into a `PyProject` object using the previously defined `pyproject.decode` function. It shows how to access parsed build system and project information. ```python import pyproject import urllib.request url = "https://raw.githubusercontent.com/Kludex/starlette/HEAD/pyproject.toml" with urllib.request.urlopen(url) as f: data = f.read() result = pyproject.decode(data) # decode the pyproject.toml print(result.build_system) print(result.project.name) ``` -------------------------------- ### Implement Post-Init Processing in Python Structs Source: https://jcristharif.com/msgspec/structs Shows how to define a `__post_init__` method in a msgspec struct for custom validation or logic after initialization, decoding, or conversion. Includes handling of exceptions. ```python import msgspec class Interval(msgspec.Struct): low: float high: float def __post_init__(self): if self.low > self.high: raise ValueError("`low` may not be greater than `high`") # Example usage: print(Interval(1, 2)) try: Interval(2, 1) except ValueError as e: print(e) try: msgspec.json.decode(b'{"low": 2, "high": 1}', type=Interval) except msgspec.ValidationError as e: print(e) ``` -------------------------------- ### Detecting Invalid pyproject.toml Schema Source: https://jcristharif.com/msgspec/_sources/examples/pyproject-toml.rst This example illustrates how msgspec catches schema validation errors, such as incorrect data types for fields. It shows a `ValidationError` when 'build-system.requires' is provided as a string instead of an array. ```python import pyproject # Assuming 'pyproject.toml' contains the invalid content: # [build-system] # requires = "hatchling" # build-backend = "hatchling.build" with open("pyproject.toml", "rb") as f: invalid = f.read() try: pyproject.decode(invalid) except Exception as e: print(e) ``` -------------------------------- ### Get Information about a Single msgspec Type Source: https://jcristharif.com/msgspec/api The `type_info` function retrieves detailed information about a single msgspec-compatible type. It returns a `Type` object representing the type's structure and constraints. For multiple types, `multi_type_info` is more efficient. ```python import msgspec # Example usage bool_type_info = msgspec.inspect.type_info(bool) print(bool_type_info) int_type_info = msgspec.inspect.type_info(int) print(int_type_info) list_int_type_info = msgspec.inspect.type_info(list[int]) print(list_int_type_info) ``` -------------------------------- ### Define Msgspec Struct Source: https://jcristharif.com/msgspec/_sources/index.rst Defines a User struct using msgspec.Struct and Python type annotations. This example shows how to declare fields with their types, including optional fields and default values, for structured data representation. ```python >>> import msgspec >>> class User(msgspec.Struct): ... """A new type describing a User""" ... name: str ... groups: set[str] = set() ... email: str | None = None ``` -------------------------------- ### Apply String Pattern Constraint with msgspec Source: https://jcristharif.com/msgspec/_sources/constraints.rst Illustrates how to apply a regular expression pattern constraint to a string during decoding with msgspec. The example shows a JSON string that fails to match the specified unanchored pattern. ```python >>> import msgspec >>> from typing import Annotated >>> msgspec.json.decode( ... b'"invalid username"', ... type=Annotated[str, msgspec.Meta(pattern="^[a-z0-9_]*$")] ... ) Traceback (most recent call last): File "", line 1, in msgspec.ValidationError: Expected `str` matching regex '^[a-z0-9_]*$' ``` -------------------------------- ### Build Memory Usage Plot with Vega-Lite Source: https://jcristharif.com/msgspec/_sources/benchmarks.rst Creates a bar chart to visualize memory usage across different libraries. It formats memory values to three decimal places and appends 'MiB' for the tooltip. This function requires the `vegaEmbed` library. ```javascript function buildMemPlot(div, rows, title) { var data = []; for (i = 0; i < rows.length; i++) { data.push({library: rows[i].label, memory: rows[i].memory}); } var spec = { "$schema": "https://vega.github.io/schema/vega-lite/v5.2.0.json", "title": title, "config": { "view": {"stroke": null}, "legend": {"title": null, "labelFontSize": 12}, "title": {"fontSize": 14, "offset": 10}, "axis": {"titleFontSize": 12, "titlePadding": 10} }, "width": "container", "data": {"values": data}, "transform": [ { "calculate": "join([format(datum.memory, '.3'), ' MiB'], '')", "as": "tooltip", } ], "mark": "bar", "encoding": { "row": { "field": "library", "header": { "orient": "left", "labelAngle": 0, "labelAlign": "left", "labelFontSize": 12 }, "sort": {"field": "memory", "order": "ascending"}, "title": null, "type": "nominal", }, "tooltip": {"field": "tooltip", "type": "nominal"}, "x": { "axis": {"grid": false, "title": "Memory (MiB)"}, "field": "memory", "type": "quantitative", }, }, }; vegaEmbed(div, spec); } ``` -------------------------------- ### Encode Data with msgspec JSON and MessagePack Source: https://jcristharif.com/msgspec/usage Demonstrates how to encode Python dictionaries into JSON and MessagePack byte strings using msgspec's respective submodules. For repeated encoding, using an Encoder instance is more efficient. ```python >>> import msgspec >>> # Encode as JSON ... msgspec.json.encode({"hello": "world"}) b'{"hello":"world"}' >>> # Encode as msgpack ... msgspec.msgpack.encode({"hello": "world"}) b'\x81\xa5hello\xa5world' ``` ```python >>> import msgspec >>> # Create a JSON encoder ... encoder = msgspec.json.Encoder() >>> # Encode as JSON using the encoder ... encoder.encode({"hello": "world"}) b'{"hello":"world"}' ``` -------------------------------- ### MessagePack Extension Type Source: https://jcristharif.com/msgspec/api Represents a MessagePack Extension Type. ```APIDOC ## MessagePack Extension Type ### Description Represents a MessagePack Extension Type, which consists of a code and associated data. ### Class `msgspec.msgpack.Ext` ### Parameters #### Initialization Parameters - **code** (int) - The integer type code for the extension. Must be between -128 and 127. - **data** (bytes, bytearray, or memoryview) - The byte buffer for the extension. ### Attributes #### code - **Type**: int - **Description**: The extension type code. #### data - **Type**: bytes, bytearray, or memoryview - **Description**: The extension data payload. ``` -------------------------------- ### Generic Types with Bounded Type Variables in msgspec Source: https://jcristharif.com/msgspec/supported-types Demonstrates how msgspec handles generic types where the type variable has a `bound`. The bound type is used when the generic type is unparametrized during decoding. Example uses `collections.abc.Sequence` as a bound. ```python import msgspec from typing import Generic, TypeVar from collections.abc import Sequence S = TypeVar("S", bound=Sequence) # Can be any sequence type class Example(msgspec.Struct, Generic[S]): value: S msg_bytes = b'{"value": [1, 2, 3]}' # These are equivalent. # The unparametrized version substitutes in `Sequence` for `S` because of the bound example_unparam = msgspec.json.decode(msg_bytes, type=Example) example_bounded = msgspec.json.decode(msg_bytes, type=Example[Sequence]) print(f"Decoded with unparametrized Example: {example_unparam}") print(f"Decoded with Example[Sequence]: {example_bounded}") ``` -------------------------------- ### Encode/Decode with NewType in msgspec Source: https://jcristharif.com/msgspec/supported-types Demonstrates how msgspec treats NewType identically to its base type for encoding and decoding JSON. Supports static analysis tools like mypy. Includes an example of validation error for incorrect types. ```python from typing import NewType UserId = NewType("UserId", int) # Encode an integer as UserId print(msgspec.json.encode(UserId(1234))) # Decode an integer into UserId print(msgspec.json.decode(b'1234', type=UserId)) # Example of a validation error when decoding a string into UserId try: msgspec.json.decode(b'"oops"', type=UserId) except msgspec.ValidationError as e: print(e) ``` -------------------------------- ### Replace Fields in a Struct Instance Source: https://jcristharif.com/msgspec/api Demonstrates how to create a new struct instance with modified field values using msgspec.structs.replace. This function takes an existing struct and keyword arguments for the fields to be changed, returning a new instance without altering the original. ```python >>> class Point(msgspec.Struct): ... x: int ... y: int >>> obj = Point(x=1, y=2) >>> msgspec.structs.replace(obj, x=3) Point(x=3, y=2) ```