Internal Models

This section describes the models used internally by Aineko to serialize and deserialize user configurations.

Pipeline Configuration

aineko.models.config_schema.Config

Pipeline configuration model.

Pipeline configurations are defined by the user in a YAML file. This model is a representation of a serialized pipeline configuration.

Source code in aineko/models/config_schema.py

class Config(BaseModel):
    """Pipeline configuration model.

    Pipeline configurations are defined by the user in a YAML file. This model
    is a representation of a serialized pipeline configuration.
    """

    class Pipeline(BaseModel):
        """Pipeline model."""

        class Node(BaseModel):
            """Node model."""

            class_name: str = Field(..., alias="class")
            node_params: Optional[dict] = None
            node_settings: Optional[NodeSettings] = None
            inputs: Optional[List[str]] = None
            outputs: Optional[List[str]] = None

        name: str
        default_node_settings: Optional[NodeSettings] = None
        nodes: Dict[str, Node]
        datasets: Dict[str, DatasetConfig]

    pipeline: Pipeline

pipeline instance-attribute

pipeline: Pipeline

Pipeline

Pipeline model.

Source code in aineko/models/config_schema.py

class Pipeline(BaseModel):
    """Pipeline model."""

    class Node(BaseModel):
        """Node model."""

        class_name: str = Field(..., alias="class")
        node_params: Optional[dict] = None
        node_settings: Optional[NodeSettings] = None
        inputs: Optional[List[str]] = None
        outputs: Optional[List[str]] = None

    name: str
    default_node_settings: Optional[NodeSettings] = None
    nodes: Dict[str, Node]
    datasets: Dict[str, DatasetConfig]

datasets instance-attribute

datasets: Dict[str, DatasetConfig]

default_node_settings class-attribute instance-attribute

default_node_settings: Optional[NodeSettings] = None

name instance-attribute

name: str

nodes instance-attribute

nodes: Dict[str, Node]

Node

Node model.

Source code in aineko/models/config_schema.py

class Node(BaseModel):
    """Node model."""

    class_name: str = Field(..., alias="class")
    node_params: Optional[dict] = None
    node_settings: Optional[NodeSettings] = None
    inputs: Optional[List[str]] = None
    outputs: Optional[List[str]] = None

class_name class-attribute instance-attribute

class_name: str = Field(..., alias='class')

inputs class-attribute instance-attribute

inputs: Optional[List[str]] = None

node_params class-attribute instance-attribute

node_params: Optional[dict] = None

node_settings class-attribute instance-attribute

node_settings: Optional[NodeSettings] = None

outputs class-attribute instance-attribute

outputs: Optional[List[str]] = None

Node Configuration

aineko.models.config_schema.NodeSettings

Node settings model.

This model is used to define settings for a node. The main use case is to define the number of CPUs to use for a node. However, it can be extended to include all Ray remote parameters as described in the Ray documentation.

Source code in aineko/models/config_schema.py

class NodeSettings(BaseModel):
    """Node settings model.

    This model is used to define settings for a node. The main use case is to
    define the number of CPUs to use for a node. However, it can be extended to
    include all Ray remote parameters as described in the [Ray documentation.](
    https://docs.ray.io/en/latest/ray-core/api/doc/ray.remote.html#ray.remote/)
    {:target="_blank"}
    """

    num_cpus: Optional[float] = Field(
        None,
        description="The number of CPUs to use for a node.",
        gt=0.0,
        examples=[1.0, 0.5],
    )
    model_config = {"extra": "allow"}

model_config class-attribute instance-attribute

model_config = {'extra': 'allow'}

num_cpus class-attribute instance-attribute

num_cpus: Optional[float] = Field(
    None,
    description="The number of CPUs to use for a node.",
    gt=0.0,
    examples=[1.0, 0.5],
)

Dataset Configuration

aineko.models.dataset_config_schema.DatasetConfig

Dataset configuration model.

Source code in aineko/models/dataset_config_schema.py

class DatasetConfig(BaseModel):
    """Dataset configuration model."""

    type: str = Field(
        ...,
        description="A dotted path to the dataset class implementation.",
        examples=[
            "aineko.datasets.kafka.KafkaDataset",
            "foo.bar.baz.BazDataset",
        ],
    )
    location: Optional[str] = Field(
        None,
        description=(
            "Location of the dataset storage layer. For example, a kafka "
            "broker address."
        ),
        examples=["localhost:9092"],
    )
    params: Optional[Dict[str, Any]] = Field(
        None,
        description=(
            "The initialization parameters for the dataset. These are mainly "
            "used by power users to fine-tune the dataset behavior. For "
            "example, Kafka topics can be configured with a number of options "
            "found in: https://kafka.apache.org/documentation.html#topicconfigs"
        ),
        examples=[{"param_1": "bar"}],
    )

location class-attribute instance-attribute

location: Optional[str] = Field(
    None,
    description="Location of the dataset storage layer. For example, a kafka broker address.",
    examples=["localhost:9092"],
)

params class-attribute instance-attribute

params: Optional[Dict[str, Any]] = Field(
    None,
    description="The initialization parameters for the dataset. These are mainly used by power users to fine-tune the dataset behavior. For example, Kafka topics can be configured with a number of options found in: https://kafka.apache.org/documentation.html#topicconfigs",
    examples=[{"param_1": "bar"}],
)

type class-attribute instance-attribute

type: str = Field(
    ...,
    description="A dotted path to the dataset class implementation.",
    examples=[
        "aineko.datasets.kafka.KafkaDataset",
        "foo.bar.baz.BazDataset",
    ],
)