diff --git a/libraries/native_model/.github/workflows/build_and_test_release.yml b/libraries/native_model/.github/workflows/build_and_test_release.yml new file mode 100644 index 00000000..35d5331a --- /dev/null +++ b/libraries/native_model/.github/workflows/build_and_test_release.yml @@ -0,0 +1,66 @@ +name: Build Test Release + +on: + push: + branches: [ main, next ] + pull_request: + branches: [ main, next ] + workflow_dispatch: + +jobs: + build_test_common_os: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + toolchain: [stable] + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6 + - name: Setup Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.toolchain }} + override: true + - uses: extractions/setup-just@v3 + - uses: hustcer/setup-nu@v3.23 + with: + version: '0.105.1' + - name: Just version + run: just --version + - name: Build + run: just build_all + - name: Test + run: just test_all + release: + name: Release + runs-on: ubuntu-latest + needs: [build_test_common_os] + if: github.ref == 'refs/heads/main' + permissions: + contents: write + packages: write + steps: + - name: Checkout code + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6 + with: + ref: main + fetch-depth: 0 + + - name: install npm + uses: actions/setup-node@v6 + with: + node-version: '20' + + - name: Semantic Release + uses: cycjimmy/semantic-release-action@v4 + with: + dry_run: ${{ github.event_name != 'workflow_dispatch' }} + extra_plugins: | + @semantic-release/commit-analyzer + @semantic-release/release-notes-generator + @semantic-release/exec + @semantic-release/github + env: + GITHUB_TOKEN: ${{ secrets.PAT_GLOBAL }} + CARGO_TOKEN: ${{ secrets.CARGO_TOKEN }} \ No newline at end of file diff --git a/libraries/native_model/.github/workflows/clippy_check.yml b/libraries/native_model/.github/workflows/clippy_check.yml new file mode 100644 index 00000000..5f35644a --- /dev/null +++ b/libraries/native_model/.github/workflows/clippy_check.yml @@ -0,0 +1,30 @@ +name: Clippy Check + +on: + push: + branches: [ main ] + pull_request: + branches: [ main, next ] + schedule: + - cron: '0 23 * * 4' + +env: + RUST_BACKTRACE: full + +jobs: + clippy_check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + - uses: extractions/setup-just@v3 + - uses: hustcer/setup-nu@v3.23 + with: + version: '0.105.1' + env: + GITHUB_TOKEN: ${{ secrets.PAT_GLOBAL }} + - name: Just version + run: just --version + - name: Clippy Check + run: just clippy_check \ No newline at end of file diff --git a/libraries/native_model/.github/workflows/conventional_commits.yml b/libraries/native_model/.github/workflows/conventional_commits.yml new file mode 100644 index 00000000..018b4506 --- /dev/null +++ b/libraries/native_model/.github/workflows/conventional_commits.yml @@ -0,0 +1,13 @@ +name: Conventional Commits + +on: + pull_request: + branches: [ main ] + +jobs: + build: + name: Conventional Commits + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6 + - uses: webiny/action-conventional-commits@v1.3.1 \ No newline at end of file diff --git a/libraries/native_model/.github/workflows/fmt_check.yml b/libraries/native_model/.github/workflows/fmt_check.yml new file mode 100644 index 00000000..e7af6e24 --- /dev/null +++ b/libraries/native_model/.github/workflows/fmt_check.yml @@ -0,0 +1,30 @@ +name: Fmt Check + +on: + push: + branches: [ main ] + pull_request: + branches: [ main, next ] + schedule: + - cron: '0 23 * * 4' + +env: + RUST_BACKTRACE: full + +jobs: + fmt_check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + - uses: extractions/setup-just@v3 + - uses: hustcer/setup-nu@v3.23 + with: + version: '0.105.1' + env: + GITHUB_TOKEN: ${{ secrets.PAT_GLOBAL }} + - name: Just version + run: just --version + - name: Fmt Check + run: just fmt_check \ No newline at end of file diff --git a/libraries/native_model/.gitignore b/libraries/native_model/.gitignore new file mode 100644 index 00000000..f7cd5926 --- /dev/null +++ b/libraries/native_model/.gitignore @@ -0,0 +1,10 @@ +/target +/Cargo.lock + +# TODO: remove it used by semantic-release/exec +node_modules/ +package-lock.json +package.json + +/native_model_macro/target +/native_model_macro/Cargo.lock diff --git a/libraries/native_model/.releaserc b/libraries/native_model/.releaserc new file mode 100644 index 00000000..f90327f3 --- /dev/null +++ b/libraries/native_model/.releaserc @@ -0,0 +1,112 @@ +{ + "branches": [ + "main" + ], + "tagFormat": "${version}", + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "releaseRules": [ + { + "breaking": true, + "release": "minor" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "perf", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "emoji": ":racehorse:", + "release": "patch" + }, + { + "emoji": ":bug:", + "release": "patch" + }, + { + "emoji": ":penguin:", + "release": "patch" + }, + { + "emoji": ":apple:", + "release": "patch" + }, + { + "emoji": ":checkered_flag:", + "release": "patch" + }, + { + "tag": "BUGFIX", + "release": "patch" + }, + { + "tag": "FEATURE", + "release": "minor" + }, + { + "tag": "SECURITY", + "release": "patch" + }, + { + "tag": "Breaking", + "release": "minor" + }, + { + "tag": "Fix", + "release": "patch" + }, + { + "tag": "Update", + "release": "minor" + }, + { + "tag": "New", + "release": "minor" + }, + { + "component": "perf", + "release": "patch" + }, + { + "component": "deps", + "release": "patch" + }, + { + "type": "FEAT", + "release": "minor" + }, + { + "type": "FIX", + "release": "patch" + } + ] + } + ], + "@semantic-release/release-notes-generator", + [ + "@semantic-release/exec", + { + "prepareCmd": "bash version_update.sh ${nextRelease.version}", + "publishCmd": "bash cargo_publish.sh" + } + ], + "@semantic-release/github" + ] +} \ No newline at end of file diff --git a/libraries/native_model/Cargo.toml b/libraries/native_model/Cargo.toml new file mode 100644 index 00000000..3ac292f6 --- /dev/null +++ b/libraries/native_model/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "native_model" +version = "0.6.4" +authors = ["Vincent Herlemont ", "quexeky "] +edition = "2021" +description = "A thin wrapper around serialized data which add information of identity and version." +license = "MIT" +repository = "https://github.com/Drop-OSS/native_model" +readme = "README.md" +keywords = ["serialization", "interoperability", "data-consistency", "flexibility", "performance"] +categories = ["data-structures", "encoding", "rust-patterns"] +rust-version = "1.73.0" + +[workspace] +members = ["native_model_macro"] + +[dependencies] +zerocopy = { version = "0.8.0", features = [ "derive"] } +thiserror = "2.0.0" +anyhow = "1.0.82" +native_model_macro = { version = "0.6.2", path = "native_model_macro" } + +serde = { version = "1.0.200", features = ["derive"], optional = true } +bincode_1_3 = { package = "bincode", version = "1.3.3", optional = true } +bincode_2 = { package = "bincode", version = "2.0", features = ["serde"], optional = true } +postcard_1_0 = { package = "postcard", version = "1.0.8", features = ["alloc"], optional = true } +rmp_serde_1_3 = { package = "rmp-serde", version = "1.3", optional = true } +doc-comment = "0.3.3" +log = "0.4.27" + +[dev-dependencies] +serde_json = "1.0.116" +criterion = { version = "0.8.0" } + +[features] +default = ["serde", "bincode_1_3"] + +[[bench]] +name = "overhead" +harness = false diff --git a/libraries/native_model/LICENSE b/libraries/native_model/LICENSE new file mode 100644 index 00000000..60a8c5ea --- /dev/null +++ b/libraries/native_model/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Vincent Herlemont + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/libraries/native_model/README.md b/libraries/native_model/README.md new file mode 100644 index 00000000..9e315b42 --- /dev/null +++ b/libraries/native_model/README.md @@ -0,0 +1,327 @@ +# Native model + +[![Crates.io](https://img.shields.io/crates/v/native_model)](https://crates.io/crates/native_model) +[![Build Test Release](https://github.com/vincent-herlemont/native_model/actions/workflows/build_and_test_release.yml/badge.svg)](https://github.com/vincent-herlemont/native_model/actions/workflows/build_and_test_release.yml) +[![Documentation](https://docs.rs/native_model/badge.svg)](https://docs.rs/native_model) +[![License](https://img.shields.io/crates/l/native_model)](LICENSE) + +Add interoperability on the top of serialization formats like bincode, postcard etc. + +See [concepts](#concepts) for more details. + +## Goals + +- **Interoperability**: Allows different applications to work together, even if they are using different + versions of the data model. +- **Data Consistency**: Ensure that we process the data expected model. +- **Flexibility**: You can use any serialization format you want. More details [here](#setup-your-serialization-format). +- **Performance**: A minimal overhead (encode: ~20 ns, decode: ~40 ps). More details [here](#performance). + +## Usage + +```text + Application 1 (DotV1) Application 2 (DotV1 and DotV2) + | | + Encode DotV1 |--------------------------------> | Decode DotV1 to DotV2 + | | Modify DotV2 + Decode DotV1 | <--------------------------------| Encode DotV2 back to DotV1 + | | +``` + + +```rust +use native_model::native_model; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Serialize, PartialEq, Debug)] +#[native_model(id = 1, version = 1)] +struct DotV1(u32, u32); + +#[derive(Deserialize, Serialize, PartialEq, Debug)] +#[native_model(id = 1, version = 2, from = DotV1)] +struct DotV2 { + name: String, + x: u64, + y: u64, +} + +impl From for DotV2 { + fn from(dot: DotV1) -> Self { + DotV2 { + name: "".to_string(), + x: dot.0 as u64, + y: dot.1 as u64, + } + } +} + +impl From for DotV1 { + fn from(dot: DotV2) -> Self { + DotV1(dot.x as u32, dot.y as u32) + } +} + +// Application 1 +let dot = DotV1(1, 2); +let bytes = native_model::encode(&dot).unwrap(); + +// Application 1 sends bytes to Application 2. + +// Application 2 +// We are able to decode the bytes directly into a new type DotV2 (upgrade). +let (mut dot, source_version) = native_model::decode::(bytes).unwrap(); +assert_eq!(dot, DotV2 { + name: "".to_string(), + x: 1, + y: 2 +}); +dot.name = "Dot".to_string(); +dot.x = 5; +// For interoperability, we encode the data with the version compatible with Application 1 (downgrade). +let bytes = native_model::encode_downgrade(dot, source_version).unwrap(); + +// Application 2 sends bytes to Application 1. + +// Application 1 +let (dot, _) = native_model::decode::(bytes).unwrap(); +assert_eq!(dot, DotV1(5, 2)); +``` + + - Full example [here](./tests_crate/tests/example/example_main.rs). + +## Serialization format + +You can use default serialization formats via the feature flags, like: + +```toml +[dependencies] +native_model = { version = "0.1", features = ["bincode_2"] } +``` + +Each feature flag corresponds to a specific minor version of the serialization format. In order to avoid breaking +changes, the default serialization format is the oldest one. + +- `bincode_1_3`: [bincode](https://docs.rs/bincode/1.3.3/bincode/) v1.3 (default) +- `bincode_2`: [bincode](https://docs.rs/bincode/2.0.0-rc.3/bincode/) v2.0.0-rc3 +- `postcard_1_0`: [postcard](https://docs.rs/postcard/1.0.0/postcard/) v1.0 +- `rpm_serde_1_3`: [rmp-serde](https://docs.rs/rmp-serde/1.3.0/rmp_serde/) v1.3 + +### Custom serialization format + +Define a struct with the name you want. This struct must implement [`native_model::Encode`](https://docs.rs/native_model/latest/native_model/trait.Encode.html) and [`native_model::Decode`](https://docs.rs/native_model/latest/native_model/trait.Decode.html) traits. + +Full examples: +- [bincode with encode/decode](./tests_crate/tests/example/custom_codec/bincode.rs) +- [bincode with serde](./tests_crate/tests/example/custom_codec/bincode_serde.rs) + +Others examples, see the default implementations: +- [bincode v1.3](./src/codec/bincode_1_3.rs) +- [bincode v2.0 (rc)](./src/codec/bincode_2.rs) +- [postcard v1.0](./src/codec/postcard_1_0.rs) +- [rmp-serde v1.3](./src/codec/rmp_serde_1_3.rs) + +### Notice +`native_model` provides implementations that rely on metadata-less formats and `serde`. +There are known issues with some `serde` advanced features such as: + +- `#[serde(flatten)]` +- `#[serde(skip)]` +- `#[serde(skip_deserializing)]` +- `#[serde(skip_serializing)]` +- `#[serde(skip_serializing_if = "path")]` +- `#[serde(tag = "...")]` +- `#[serde(untagged)]` + +Or types implementing similar strategies such as [`serde_json::Value`][serde_json_value]. + +The `rmp-serde` serialization format can optionally support them serializing structs as maps, the `RmpSerdeNamed` struct is provided to support this use-case. + +[serde_json_value]: https://docs.rs/serde_json/latest/serde_json/enum.Value.html + +## Data model + +Define your model using the macro [`native_model`](file:///home/vincentherlemont/IdeaProjects/native_model/target/doc/native_model/attr.native_model.html). + +Attributes: +- `id = u32`: The unique identifier of the model. +- `version = u32`: The version of the model. +- `with = type`: The serialization format that you use for the Encode/Decode implementation. Setup [here](#setup-your-serialization-format). +- `from = type`: Optional, the previous version of the model. + - `type`: The previous version of the model that you use for the From implementation. +- `try_from = (type, error)`: Optional, the previous version of the model with error handling. + - `type`: The previous version of the model that you use for the TryFrom implementation. + - `error`: The error type that you use for the TryFrom implementation. + +```rust +use native_model::native_model; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Serialize, PartialEq, Debug)] +#[native_model(id = 1, version = 1)] +struct DotV1(u32, u32); + +#[derive(Deserialize, Serialize, PartialEq, Debug)] +#[native_model(id = 1, version = 2, from = DotV1)] +struct DotV2 { + name: String, + x: u64, + y: u64, +} + +// Implement the conversion between versions From for DotV2 and From for DotV1. + +impl From for DotV2 { + fn from(dot: DotV1) -> Self { + DotV2 { + name: "".to_string(), + x: dot.0 as u64, + y: dot.1 as u64, + } + } +} + +impl From for DotV1 { + fn from(dot: DotV2) -> Self { + DotV1(dot.x as u32, dot.y as u32) + } +} + +#[derive(Deserialize, Serialize, PartialEq, Debug)] +#[native_model(id = 1, version = 3, try_from = (DotV2, anyhow::Error))] +struct DotV3 { + name: String, + cord: Cord, +} + +#[derive(Deserialize, Serialize, PartialEq, Debug)] +struct Cord { + x: u64, + y: u64, +} + +// Implement the conversion between versions From for DotV3 and From for DotV2. + +impl TryFrom for DotV3 { + type Error = anyhow::Error; + + fn try_from(dot: DotV2) -> Result { + Ok(DotV3 { + name: dot.name, + cord: Cord { x: dot.x, y: dot.y }, + }) + } +} + +impl TryFrom for DotV2 { + type Error = anyhow::Error; + + fn try_from(dot: DotV3) -> Result { + Ok(DotV2 { + name: dot.name, + x: dot.cord.x, + y: dot.cord.y, + }) + } +} + +``` + +## Codecs + +`native_model` comes with several optional built-in serializer features available: + +- [bincode 1.3](https://crates.io/crates/bincode/1.3.3) + - This is the default codec. + - **Warning: This codec may not work with all serde-derived types.** + +- [bincode 2.0.0-rc.3](https://crates.io/crates/bincode/2.0.0-rc.3) + - Enable the `bincode_2` feature and use the `native_model::bincode_2::Bincode` attribute to have `native_db` use this crate for serializing & deserializing. + - **Warning: This codec may not work with all serde-derived types.** + +- [postcard 1.0](https://crates.io/crates/postcard/1.0.8) + - Enable the `postcard_1_0` feature and use the `native_model::postcard_1_0::PostCard` attribute. + - **Warning: This codec may not work with all serde-derived types.** + +- [rmp-serde 1.3](https://crates.io/crates/rmp-serde/1.3.0) + - Enable the `rmp_serde_1_3` feature and use the `native_model::rmp_serde_1_3::RmpSerde` attribute. + +###### Codec example: + +As example, to use `rmp-serde`: + +1. In your project's `Cargo.toml` file, enable the `rmp_serde_1_3` feature for the `native_model` dependency. + - Be sure to check `crates.io` for the most recent [`native_model`](https://crates.io/crates/native_model) version number. + +```toml +[dependencies] +serde = { version = "1.0", features = [ "derive" ] } +native_model = { version = "0.4", features = [ "rmp_serde_1_3" ] } +``` + +2. Assign the `rmp_serde_1_3` codec to your `struct` using the `with` attribute: + +```rust +use native_model::native_model; + +#[derive(Clone, Default, serde::Deserialize, serde::Serialize)] +#[native_model(id = 1, version = 1, with = native_model::rmp_serde_1_3::RmpSerde)] +struct MyStruct { + my_string: String, + // etc. +} +``` + +###### Additional reading + +You may also want to check out [David Koloski](https://github.com/djkoloski)'s [Rust serialization benchmarks](https://github.com/djkoloski/rust_serialization_benchmark) for help selecting the codec (i.e. `bincode_1_3`, `rmp_serde_1_3`, etc.) that's best for your project. + +## Status + +Early development. Not ready for production. + +## Concepts + +In order to understand how the native model works, you need to understand the following concepts. + +- **Identity**(`id`): The identity is the unique identifier of the model. It is used to identify the model and + prevent to decode a model into the wrong Rust type. +- **Version**(`version`) The version is the version of the model. It is used to check the compatibility between two + models. +- **Encode**: The encode is the process of converting a model into a byte array. +- **Decode**: The decode is the process of converting a byte array into a model. +- **Downgrade**: The downgrade is the process of converting a model into a previous version of the model. +- **Upgrade**: The upgrade is the process of converting a model into a newer version of the model. + +Under the hood, the native model is a thin wrapper around serialized data. The `id` and the `version` are twice encoded with a [`little_endian::U32`](https://docs.rs/zerocopy/latest/zerocopy/byteorder/little_endian/type.U32.html). That represents 8 bytes, that are added at the beginning of the data. + +``` text ++------------------+------------------+------------------------------------+ +| ID (4 bytes) | Version (4 bytes)| Data (indeterminate-length bytes) | ++------------------+------------------+------------------------------------+ +``` + +Full example [here](tests/example/example_define_model.rs). + +## Performance + +Native model has +been designed to have a minimal and constant overhead. That means that the overhead is the same +whatever the size of the data. Under the hood we use the [zerocopy](https://docs.rs/zerocopy/latest/zerocopy/) crate +to avoid unnecessary copies. + +👉 To know the total time of the encode/decode, you need to add the time of your serialization format. + +Resume: +- **Encode**: ~20 ns +- **Decode**: ~40 ps + +| data size | encode time (ns) | decode time (ps) | +|:--------------------:|:---------------------:|:-----------------------:| +| 1 B | 19.769 ns - 20.154 ns | 40.526 ps - 40.617 ps | +| 1 KiB | 19.597 ns - 19.971 ns | 40.534 ps - 40.633 ps | +| 1 MiB | 19.662 ns - 19.910 ns | 40.508 ps - 40.632 ps | +| 10 MiB | 19.591 ns - 19.980 ns | 40.504 ps - 40.605 ps | +| 100 MiB | 19.669 ns - 19.867 ns | 40.520 ps - 40.644 ps | + +Benchmark of the native model overhead [here](benches/overhead.rs). + diff --git a/libraries/native_model/benches/overhead.rs b/libraries/native_model/benches/overhead.rs new file mode 100644 index 00000000..75ea7d0e --- /dev/null +++ b/libraries/native_model/benches/overhead.rs @@ -0,0 +1,41 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use native_model::Model; +use native_model_macro::native_model; +use serde::{Deserialize, Serialize}; +#[derive(Serialize, Deserialize)] +#[native_model(id = 1, version = 1)] +struct Data(Vec); + +fn wrap(data: &mut Vec) { + native_model::wrapper::native_model_encode(data, 1, 1); +} + +fn unwrap(data: &mut Vec) { + native_model::wrapper::Wrapper::deserialize(&data[..]).unwrap(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let mut group = c.benchmark_group("encode"); + + // 1 byte, 1KB, 1MB, 10MB, 100MB + for nb_bytes in [1, 1024, 1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024].into_iter() { + group.throughput(criterion::Throughput::Bytes(nb_bytes as u64)); + + // encode + let data = Data(vec![1; nb_bytes]); + let mut encode_body = data.native_model_encode_body().unwrap(); + group.bench_function(BenchmarkId::new("encode", nb_bytes), |b| { + b.iter(|| wrap(&mut encode_body)) + }); + + // decode + let data = Data(vec![1; nb_bytes]); + let mut encode_body = native_model::encode(&data).unwrap(); + group.bench_function(BenchmarkId::new("decode", nb_bytes), |b| { + b.iter(|| unwrap(&mut encode_body)) + }); + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/libraries/native_model/cargo_publish.sh b/libraries/native_model/cargo_publish.sh new file mode 100755 index 00000000..36787b98 --- /dev/null +++ b/libraries/native_model/cargo_publish.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +set -e +set -x + +ARG_TOKEN="--token=$CARGO_TOKEN" + +cd $DIR/native_model_macro +cargo publish $ARG_TOKEN $@ + +cd $DIR +cargo publish $ARG_TOKEN $@ \ No newline at end of file diff --git a/libraries/native_model/justfile b/libraries/native_model/justfile new file mode 100644 index 00000000..78f7fa93 --- /dev/null +++ b/libraries/native_model/justfile @@ -0,0 +1,79 @@ +set shell := ["nu", "-c"] + +default: + @just --list --unsorted; + +build_no_default: + cargo build --no-default-features + +build_default: + cargo build + +build_serde: + cargo build --no-default-features --features serde + +build_bincode_1_3: + cargo build --features bincode_1_3 + +build_no_default_bincode_1_3: + cargo build --no-default-features --features serde --features bincode_1_3 + +build_bincode_2: + cargo build --features bincode_2 + +build_no_default_bincode_2: + cargo build --no-default-features --features serde --features bincode_2 + +build_postcard_1_0: + cargo build --features postcard_1_0 + +build_no_default_postcard_1_0: + cargo build --no-default-features --features serde --features postcard_1_0 + +build_all: build_no_default build_default build_serde build_bincode_1_3 build_no_default_bincode_1_3 build_bincode_2 build_no_default_bincode_2 build_postcard_1_0 build_no_default_postcard_1_0 + +_tests_crate args='': + cd tests_crate; \ + cargo test {{args}} + +test_no_default: + @just _tests_crate '--no-default-features' + +test_default: + @just _tests_crate args='' + +test_bincode_1_3: + @just _tests_crate '--features bincode_1_3' + +test_bincode_2: + @just _tests_crate '--features bincode_2' + +test_postcard_1_0: + @just _tests_crate '--features postcard_1_0' + +test_docs: + cargo test --doc --all-features + +test_all: test_docs test_no_default test_default test_bincode_1_3 test_bincode_2 test_postcard_1_0 + +bench_overhead: + cargo bench --bench overhead + +bench_all: bench_overhead + +format: + cargo clippy; \ + cargo fmt --all + +fmt_check: + cargo fmt --all -- --check + +clippy_check: + rustc --version; \ + cargo clippy --version; \ + cargo clippy -- -D warnings + +# Format check +fc: + just fmt_check; \ + just clippy_check diff --git a/libraries/native_model/native_model_macro/Cargo.toml b/libraries/native_model/native_model_macro/Cargo.toml new file mode 100644 index 00000000..fd89088c --- /dev/null +++ b/libraries/native_model/native_model_macro/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "native_model_macro" +version = "0.6.4" +authors = ["Vincent Herlemont ", "quexeky "] +edition = "2018" +description = "A procedural macro for native_model" +license = "MIT" +repository = "https://github.com/Drop-OSS/native_model" +readme = "README.md" + + +[lib] +path = "src/lib.rs" +proc-macro = true + +[dependencies] +syn = { version = "2.0.60", features = ["full"] } +quote = "1.0.36" +proc-macro2 = "1.0.81" +log = "0.4.27" diff --git a/libraries/native_model/native_model_macro/README.md b/libraries/native_model/native_model_macro/README.md new file mode 100644 index 00000000..45836a41 --- /dev/null +++ b/libraries/native_model/native_model_macro/README.md @@ -0,0 +1 @@ +A procedural macro for [native_model](https://github.com/Drop-OSS/native_model). \ No newline at end of file diff --git a/libraries/native_model/native_model_macro/src/lib.rs b/libraries/native_model/native_model_macro/src/lib.rs new file mode 100644 index 00000000..f4bce320 --- /dev/null +++ b/libraries/native_model/native_model_macro/src/lib.rs @@ -0,0 +1,127 @@ +extern crate proc_macro; +extern crate log; + +mod method; + +use crate::method::{ + generate_native_model_decode_body, generate_native_model_decode_upgrade_body, + generate_native_model_encode_body, + generate_native_model_id, generate_native_model_version, +}; +use proc_macro::TokenStream; +use quote::quote; +use syn::meta::ParseNestedMeta; +use syn::parse::{Parse, Result}; +use syn::punctuated::Punctuated; +use syn::token; +use syn::{parse_macro_input, DeriveInput, LitInt, Path, Token}; + +// Inspiration: https://docs.rs/syn/2.0.29/syn/meta/fn.parser.html#example-1 +pub(crate) struct ModelAttributes { + pub(crate) id: Option, + pub(crate) version: Option, + // type + pub(crate) with: Option, + // type + pub(crate) from: Option, + // (type, try_from::Error type) + pub(crate) try_from: Option<(Path, Path)>, +} + +impl Default for ModelAttributes { + fn default() -> Self { + ModelAttributes { + id: None, + version: None, + with: Some(syn::parse_str::("native_model::bincode_1_3::Bincode").unwrap()), + from: None, + try_from: None, + } + } +} + +impl ModelAttributes { + fn parse(&mut self, meta: ParseNestedMeta) -> Result<()> { + if meta.path.is_ident("id") { + self.id = Some(meta.value()?.parse()?); + } else if meta.path.is_ident("version") { + self.version = Some(meta.value()?.parse()?); + } else if meta.path.is_ident("with") { + self.with = Some(meta.value()?.parse()?); + } else if meta.path.is_ident("from") { + self.from = Some(meta.value()?.parse()?); + } else if meta.path.is_ident("try_from") { + let tuple_try_from: TupleTryFrom = meta.value()?.parse()?; + let mut fields = tuple_try_from.fields.into_iter(); + self.try_from.replace(( + fields.next().unwrap().clone(), + fields.next().unwrap().clone(), + )); + } else { + panic!("Unknown attribute: {}", meta.path.get_ident().unwrap()); + } + Ok(()) + } +} + +#[derive(Default)] +pub(crate) struct TupleTryFrom { + pub(crate) _parent_token: token::Paren, + pub(crate) fields: Punctuated, +} + +impl Parse for TupleTryFrom { + fn parse(input: syn::parse::ParseStream) -> syn::Result { + let content; + Ok(TupleTryFrom { + _parent_token: syn::parenthesized!(content in input), + fields: content.parse_terminated(Path::parse, Token![,])?, + }) + } +} + +/// Macro which add identity and version to your rust type. +/// +/// Attributes: +/// - `id = u32`: The unique identifier of the model. +/// - `version = u32`: The version of the model. +/// - `with` = type: Required, the serialization/deserialization library that you use. Must implement `native_model::Encode` and `native_model::Decode`. +/// - `from = type`: Optional, the previous version of the model. +/// - `type`: The previous version of the model that you use for the From implementation. +/// - `try_from = (type, error)`: Optional, the previous version of the model with error handling. +/// - `type`: The previous version of the model that you use for the TryFrom implementation. +/// - `error`: The error type that you use for the TryFrom implementation. +/// +/// See examples: +/// - [Setup your data model](https://github.com/vincent-herlemont/native_model_private#setup-your-data-model). +/// - other [examples](https://github.com/Drop-OSS/native_model/tree/master/tests/example) +#[proc_macro_attribute] +pub fn native_model(args: TokenStream, input: TokenStream) -> TokenStream { + let ast = parse_macro_input!(input as DeriveInput); + let struct_name = &ast.ident; + + let mut attrs = ModelAttributes::default(); + let model_attributes_parser = syn::meta::parser(|meta| attrs.parse(meta)); + parse_macro_input!(args with model_attributes_parser); + + let native_model_id_fn = generate_native_model_id(&attrs); + let native_model_version_fn = generate_native_model_version(&attrs); + let native_model_encode_body_fn = generate_native_model_encode_body(&attrs); + let native_model_decode_body_fn = generate_native_model_decode_body(&attrs); + let native_model_decode_upgrade_body_fn = generate_native_model_decode_upgrade_body(&attrs, struct_name); + + + let gen = quote! { + #ast + + impl native_model::Model for #struct_name { + #native_model_id_fn + #native_model_version_fn + #native_model_encode_body_fn + #native_model_decode_body_fn + #native_model_decode_upgrade_body_fn + } + }; + + gen.into() +} diff --git a/libraries/native_model/native_model_macro/src/method/decode_body.rs b/libraries/native_model/native_model_macro/src/method/decode_body.rs new file mode 100644 index 00000000..e1a1195e --- /dev/null +++ b/libraries/native_model/native_model_macro/src/method/decode_body.rs @@ -0,0 +1,23 @@ +use crate::ModelAttributes; +use proc_macro2::TokenStream; +use quote::quote; + +pub(crate) fn generate_native_model_decode_body(attrs: &ModelAttributes) -> TokenStream { + let id = attrs.id.clone().expect("`id` is required"); + let with = attrs.with.clone().expect("`with` is required"); + let gen = quote! { + fn native_model_decode_body(data: Vec, id: u32) -> std::result::Result { + if id != #id { + return Err(native_model::DecodeBodyError::MismatchedModelId); + } + + use native_model::Decode; + #with::decode(data).map_err(|e| native_model::DecodeBodyError::DecodeError { + msg: format!("{}", e), + source: e.into(), + }) + } + }; + + gen +} diff --git a/libraries/native_model/native_model_macro/src/method/decode_upgrade_body.rs b/libraries/native_model/native_model_macro/src/method/decode_upgrade_body.rs new file mode 100644 index 00000000..7997ae83 --- /dev/null +++ b/libraries/native_model/native_model_macro/src/method/decode_upgrade_body.rs @@ -0,0 +1,56 @@ +use crate::ModelAttributes; +use proc_macro2::TokenStream; +use quote::quote; +use syn::Ident; + +pub(crate) fn generate_native_model_decode_upgrade_body(attrs: &ModelAttributes, struct_name: &Ident) -> TokenStream { + let native_model_from = attrs.from.clone(); + let native_model_try_from = attrs.try_from.clone(); + + let name = struct_name.to_string(); + + let model_from_or_try_from = if let Some(from) = native_model_from { + quote! { + ::log::info!("Upgrading database {} from version {} to version {}", #name, #from::native_model_version(), Self::native_model_version()); + + #from::native_model_decode_upgrade_body(data, id, version).map(|a| a.into()) + } + } else if let Some((try_from, error_try_from)) = native_model_try_from { + quote! { + ::log::info!("Attempting to upgrade database {} from version {} to version {}", #name, #try_from::native_model_version(), Self::native_model_version()); + let result = #try_from::native_model_decode_upgrade_body(data, id, version).map(|b| { + b.try_into() + .map_err(|e: #error_try_from| native_model::UpgradeError { + msg: format!("{}", e), + source: e.into(), + }) + })??; + Ok(result) + } + } else { + quote! { + Err(native_model::Error::UpgradeNotSupported { + from: version, + to: Self::native_model_version(), + }) + } + }; + + let gen = quote! { + fn native_model_decode_upgrade_body(data: Vec, id: u32, version: u32) -> native_model::Result { + if version == Self::native_model_version() { + let result = Self::native_model_decode_body(data, id)?; + Ok(result) + } else if version < Self::native_model_version() { + #model_from_or_try_from + } else { + Err(native_model::Error::UpgradeNotSupported { + from: version, + to: Self::native_model_version(), + }) + } + } + }; + + gen +} diff --git a/libraries/native_model/native_model_macro/src/method/encode_body.rs b/libraries/native_model/native_model_macro/src/method/encode_body.rs new file mode 100644 index 00000000..6cff9ee7 --- /dev/null +++ b/libraries/native_model/native_model_macro/src/method/encode_body.rs @@ -0,0 +1,18 @@ +use crate::ModelAttributes; +use proc_macro2::TokenStream; +use quote::quote; + +pub(crate) fn generate_native_model_encode_body(attrs: &ModelAttributes) -> TokenStream { + let with = attrs.with.clone().expect("`with` is required"); + let gen = quote! { + fn native_model_encode_body(&self) -> std::result::Result, native_model::EncodeBodyError> { + use native_model::Encode; + #with::encode(self).map_err(|e| native_model::EncodeBodyError { + msg: format!("{}", e), + source: e.into(), + }) + } + }; + + gen +} diff --git a/libraries/native_model/native_model_macro/src/method/id.rs b/libraries/native_model/native_model_macro/src/method/id.rs new file mode 100644 index 00000000..bc697d9f --- /dev/null +++ b/libraries/native_model/native_model_macro/src/method/id.rs @@ -0,0 +1,17 @@ +use crate::ModelAttributes; +use proc_macro2::TokenStream; +use quote::quote; + +pub(crate) fn generate_native_model_id(model_attributes: &ModelAttributes) -> TokenStream { + let native_model_id = model_attributes.id.clone().unwrap(); + let gen = quote! { + fn native_model_id() -> u32 { + #native_model_id + } + + fn native_model_id_str() -> &'static str { + stringify!(#native_model_id) + } + }; + gen +} diff --git a/libraries/native_model/native_model_macro/src/method/mod.rs b/libraries/native_model/native_model_macro/src/method/mod.rs new file mode 100644 index 00000000..fedf220a --- /dev/null +++ b/libraries/native_model/native_model_macro/src/method/mod.rs @@ -0,0 +1,11 @@ +mod decode_body; +mod decode_upgrade_body; +mod encode_body; +mod id; +mod version; + +pub(crate) use decode_body::*; +pub(crate) use decode_upgrade_body::*; +pub(crate) use encode_body::*; +pub(crate) use id::*; +pub(crate) use version::*; diff --git a/libraries/native_model/native_model_macro/src/method/version.rs b/libraries/native_model/native_model_macro/src/method/version.rs new file mode 100644 index 00000000..f4dac013 --- /dev/null +++ b/libraries/native_model/native_model_macro/src/method/version.rs @@ -0,0 +1,17 @@ +use crate::ModelAttributes; +use proc_macro2::TokenStream; +use quote::quote; + +pub(crate) fn generate_native_model_version(model_attributes: &ModelAttributes) -> TokenStream { + let native_model_version = model_attributes.version.clone().unwrap(); + let gen = quote! { + fn native_model_version() -> u32 { + #native_model_version + } + + fn native_model_version_str() -> &'static str { + stringify!(#native_model_version) + } + }; + gen +} diff --git a/libraries/native_model/renovate.json b/libraries/native_model/renovate.json new file mode 100644 index 00000000..efa975ad --- /dev/null +++ b/libraries/native_model/renovate.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ], + "semanticCommits": "enabled", + "semanticCommitType": "chore", + "semanticCommitScope": "deps", + "platformAutomerge": true, + "packageRules": [ + { + "description": "Automerge non-major updates", + "matchUpdateTypes": [ + "minor", + "patch" + ], + "automerge": true + }, + { + "description": "Automerge actions", + "matchDepTypes": [ + "action" + ], + "matchUpdateTypes": [ + "major", + "minor", + "patch" + ], + "automerge": true + } + ], + "regexManagers": [ + { + "fileMatch": [ + "^\\.github/workflows/[^/]+\\.ya?ml$" + ], + "matchStrings": [ + "uses: hustcer/setup-nu@.*?\\n.*?version: '\\s*(?.*?)'" + ], + "depNameTemplate": "nushell", + "datasourceTemplate": "github-releases", + "packageNameTemplate": "nushell/nushell" + } + ] +} \ No newline at end of file diff --git a/libraries/native_model/src/codec/bincode_1_3.rs b/libraries/native_model/src/codec/bincode_1_3.rs new file mode 100644 index 00000000..5010422d --- /dev/null +++ b/libraries/native_model/src/codec/bincode_1_3.rs @@ -0,0 +1,52 @@ +//! [bincode 1.3](https://crates.io/crates/bincode/1.3.3) · +//! The default codec for serializing & deserializing. + +/// Used to specify that the +/// [bincode 1.3](https://crates.io/crates/bincode/1.3.3) crate is to be used +/// for serialization & deserialization. +/// +/// # Warning +/// +/// `bincode` [does not implement](https://github.com/bincode-org/bincode/issues/548) +/// all [serde](https://crates.io/crates/serde) features. Errors may be +/// encountered when using this with some types. +/// +/// If you are encountering errors when using this codec on your types, try +/// using the `rmp_serde_1_3` codec instead. +/// +/// # Basic usage +/// +/// Use the [`with`](crate::native_model) attribute on your type to instruct +/// `native_model` to use `Bincode` for serialization & deserialization. +/// +/// Example usage: +/// +/// ```rust +/// # use native_model::*; +/// #[derive(Clone, Default, serde::Deserialize, serde::Serialize)] +/// #[native_model(id = 1, version = 1, with = native_model::bincode_1_3::Bincode)] +/// struct MyStruct { +/// my_string: String +/// } +/// ``` + +#[derive(Default)] +pub struct Bincode; + +#[cfg(all(feature = "serde", feature = "bincode_1_3"))] +impl super::Encode for Bincode { + type Error = bincode_1_3::Error; + /// Serializes a type into bytes using the `bincode` `1.3` crate. + fn encode(obj: &T) -> Result, Self::Error> { + bincode_1_3::serialize(obj) + } +} + +#[cfg(all(feature = "serde", feature = "bincode_1_3"))] +impl serde::Deserialize<'de>> super::Decode for Bincode { + type Error = bincode_1_3::Error; + /// Deserializes a type from bytes using the `bincode` `1.3` crate. + fn decode(data: Vec) -> Result { + bincode_1_3::deserialize(&data[..]) + } +} diff --git a/libraries/native_model/src/codec/bincode_2.rs b/libraries/native_model/src/codec/bincode_2.rs new file mode 100644 index 00000000..5b89978c --- /dev/null +++ b/libraries/native_model/src/codec/bincode_2.rs @@ -0,0 +1,54 @@ +//! [bincode 2.0](https://crates.io/crates/bincode/2.0.1) · +//! Enable the `bincode_2` feature and annotate your type with +//! `native_model::bincode_2::Bincode` to have `native_db` use this crate for +//! serializing & deserializing. + +/// Used to specify the +/// [bincode 2.0](https://crates.io/crates/bincode/2.0.1) +/// crate for serialization & deserialization. +/// +/// # Warning +/// +/// `bincode` [does not implement](https://docs.rs/bincode/2.0.1/bincode/serde/index.html#known-issues) +/// all [serde](https://crates.io/crates/serde) features. Errors may be +/// encountered when using this with some types. +/// +/// If you are encountering errors when using this codec on your types, try +/// using the `rmp_serde_1_3` codec instead. +/// +/// # Basic usage +/// +/// After enabling the `bincode_2` feature in your `Cargo.toml`, use the +/// [`with`](crate::native_model) attribute on your type to instruct +/// `native_model` to use `Bincode` for serialization & deserialization. +/// +/// Example usage: +/// +/// ```rust +/// # use native_model::*; +/// #[derive(Clone, Default, serde::Deserialize, serde::Serialize)] +/// #[native_model(id = 1, version = 1, with = native_model::bincode_2::Bincode)] +/// struct MyStruct { +/// my_string: String +/// } +/// ``` + +pub struct Bincode; + +#[cfg(all(feature = "serde", feature = "bincode_2"))] +impl super::Encode for Bincode { + type Error = bincode_2::error::EncodeError; + /// Serializes a type into bytes using the `bincode` `2.0` crate. + fn encode(obj: &T) -> Result, Self::Error> { + bincode_2::serde::encode_to_vec(obj, bincode_2::config::standard()) + } +} + +#[cfg(all(feature = "serde", feature = "bincode_2"))] +impl serde::Deserialize<'de>> super::Decode for Bincode { + type Error = bincode_2::error::DecodeError; + /// Deserializes a type from bytes using the `bincode` `2.0` crate. + fn decode(data: Vec) -> Result { + Ok(bincode_2::serde::decode_from_slice(&data, bincode_2::config::standard())?.0) + } +} diff --git a/libraries/native_model/src/codec/mod.rs b/libraries/native_model/src/codec/mod.rs new file mode 100644 index 00000000..e834bd48 --- /dev/null +++ b/libraries/native_model/src/codec/mod.rs @@ -0,0 +1,62 @@ +//! Traits and implementations for encoding types into a series of bytes and +//! decoding bytes back into types. + +#[cfg(any(all(feature = "serde", feature = "bincode_1_3"), doc))] +pub mod bincode_1_3; +#[cfg(any(all(feature = "serde", feature = "bincode_2"), doc))] +pub mod bincode_2; +#[cfg(any(all(feature = "serde", feature = "postcard_1_0"), doc))] +pub mod postcard_1_0; +#[cfg(any(all(feature = "serde", feature = "rmp_serde_1_3"), doc))] +pub mod rmp_serde_1_3; + +/// Encode trait for your own encoding method. +/// +/// Example: +/// ```rust +/// use bincode_2::{error::EncodeError,serde::encode_to_vec, config::standard}; +/// use serde::Serialize; +/// pub struct Bincode; +/// +/// impl native_model::Encode for Bincode { +/// type Error = EncodeError; +/// fn encode(obj: &T) -> Result, EncodeError> { +/// Ok(encode_to_vec(&obj, standard())?) +/// } +/// } +/// ``` +pub trait Encode { + type Error; + /// Encodes a `T` type into a series of bytes. + /// + /// # Errors + /// + /// The errors returned from this function depend on the trait implementor + /// (the serializer), i.e. `bincode_2`. + fn encode(obj: &T) -> Result, Self::Error>; +} + +/// Decode trait for your own decoding method. +/// +/// Example: +/// ```rust +/// use bincode_2::{error::DecodeError,serde::decode_from_slice, config::standard}; +/// use serde::Deserialize; +/// pub struct Bincode; +/// +/// impl Deserialize<'a>> native_model::Decode for Bincode { +/// type Error = DecodeError; +/// fn decode(data: Vec) -> Result { +/// Ok(decode_from_slice(&data, standard())?.0) +/// } +/// } +pub trait Decode { + type Error; + /// Decodes a series of bytes back into a `T` type. + /// + /// # Errors + /// + /// The errors returned from this function depend on the trait implementor + /// (the deserializer), i.e. `bincode_2`. + fn decode(data: Vec) -> Result; +} diff --git a/libraries/native_model/src/codec/postcard_1_0.rs b/libraries/native_model/src/codec/postcard_1_0.rs new file mode 100644 index 00000000..a44b16dc --- /dev/null +++ b/libraries/native_model/src/codec/postcard_1_0.rs @@ -0,0 +1,51 @@ +//! [postcard 1.0](https://crates.io/crates/postcard/1.0.8) · +//! Enable the `postcard_1_0` feature and annotate your type with +//! `native_model::postcard_1_0::PostCard` to have `native_db` use this crate. + +/// Used to specify the [postcard 1.0](https://crates.io/crates/postcard/1.0.8) +/// crate for serialization & deserialization. +/// +/// # Warning +/// +/// `postcard` does not implement all [serde](https://crates.io/crates/serde) +/// features. Errors may be encountered when using this with some types. +/// +/// If you are encountering errors when using this codec on your types, try +/// using the `rmp_serde_1_3` codec instead. +/// +/// # Basic usage +/// +/// After enabling the `postcard_1_0` feature in your `Cargo.toml`, use the +/// [`with`](crate::native_model) attribute on your type to instruct +/// `native_model` to use `PostCard` for serialization & deserialization. +/// +/// Example usage: +/// +/// ```rust +/// # use native_model::*; +/// #[derive(Clone, Default, serde::Deserialize, serde::Serialize)] +/// #[native_model(id = 1, version = 1, with = native_model::postcard_1_0::PostCard)] +/// struct MyStruct { +/// my_string: String +/// } +/// ``` + +pub struct PostCard; + +#[cfg(all(feature = "serde", feature = "postcard_1_0"))] +impl super::Encode for PostCard { + type Error = postcard_1_0::Error; + /// Serializes a type into bytes using the `postcard` `1.0` crate. + fn encode(obj: &T) -> Result, Self::Error> { + postcard_1_0::to_allocvec(obj) + } +} + +#[cfg(all(feature = "serde", feature = "postcard_1_0"))] +impl serde::Deserialize<'de>> super::Decode for PostCard { + type Error = postcard_1_0::Error; + /// Deserializes a type from bytes using the `postcard` `1.0` crate. + fn decode(data: Vec) -> Result { + postcard_1_0::from_bytes(&data) + } +} diff --git a/libraries/native_model/src/codec/rmp_serde_1_3.rs b/libraries/native_model/src/codec/rmp_serde_1_3.rs new file mode 100644 index 00000000..8cbba4a3 --- /dev/null +++ b/libraries/native_model/src/codec/rmp_serde_1_3.rs @@ -0,0 +1,91 @@ +//! [rmp-serde 1.3](https://crates.io/crates/rmp-serde/1.3.0) · +//! Enable the `rmp_serde_1_3` feature and +//! [`annotate your type`](crate::native_model) with +//! `native_model::rmp_serde_1_3::RmpSerde` or `native_model::rmp_serde_1_3::RmpSerdeNamed` +//! to have `native_db` use this crate. + +/// Used to specify the +/// [rmp-serde 1.3](https://crates.io/crates/rmp-serde/1.3.0) +/// crate for serialization & deserialization, using arrays to serialize structs. +/// +/// Do not use this if you plan to use serde features that skip serializing fields, +/// use [RmpSerdeNamed] instead. +/// +/// # Basic usage +/// +/// After enabling the `rmp_serde_1_3` feature in your `Cargo.toml`, use the +/// [`with`](crate::native_model) attribute on your type to instruct +/// `native_model` to use `RmpSerde` for serialization & deserialization. +/// +/// Example usage: +/// +/// ```rust +/// # use native_model::*; +/// #[derive(Clone, Default, serde::Deserialize, serde::Serialize)] +/// #[native_model(id = 1, version = 1, with = native_model::rmp_serde_1_3::RmpSerde)] +/// struct MyStruct { +/// my_string: String +/// } +/// ``` + +pub struct RmpSerde; + +#[cfg(all(feature = "serde", feature = "rmp_serde_1_3"))] +impl crate::Encode for RmpSerde { + type Error = rmp_serde_1_3::encode::Error; + /// Serializes a type into bytes using the `rmp-serde` `1.3` crate. + fn encode(obj: &T) -> Result, Self::Error> { + rmp_serde_1_3::encode::to_vec(obj) + } +} + +#[cfg(all(feature = "serde", feature = "rmp_serde_1_3"))] +impl serde::Deserialize<'de>> crate::Decode for RmpSerde { + type Error = rmp_serde_1_3::decode::Error; + /// Deserializes a type from bytes using the `rmp-serde` `1.3` crate. + fn decode(data: Vec) -> Result { + rmp_serde_1_3::decode::from_slice(&data) + } +} + +/// Used to specify the +/// [rmp-serde 1.3](https://crates.io/crates/rmp-serde/1.3.0) +/// crate for serialization & deserialization, using maps to serialize structs. +/// +/// # Basic usage +/// +/// After enabling the `rmp_serde_1_3` feature in your `Cargo.toml`, use the +/// [`with`](crate::native_model) attribute on your type to instruct +/// `native_model` to use `RmpSerdeNamed` for serialization & deserialization. +/// +/// Example usage: +/// +/// ```rust +/// # use native_model::*; +/// #[derive(Clone, Default, serde::Deserialize, serde::Serialize)] +/// #[native_model(id = 1, version = 1, with = native_model::rmp_serde_1_3::RmpSerdeNamed)] +/// struct MyStruct { +/// #[serde(skip_serializing_if = "String::is_empty")] +/// my_string: String +/// } +/// ``` + +pub struct RmpSerdeNamed; + +#[cfg(all(feature = "serde", feature = "rmp_serde_1_3"))] +impl crate::Encode for RmpSerdeNamed { + type Error = rmp_serde_1_3::encode::Error; + /// Serializes a type into bytes using the `rmp-serde` `1.3` crate. + fn encode(obj: &T) -> Result, Self::Error> { + rmp_serde_1_3::encode::to_vec_named(obj) + } +} + +#[cfg(all(feature = "serde", feature = "rmp_serde_1_3"))] +impl serde::Deserialize<'de>> crate::Decode for RmpSerdeNamed { + type Error = rmp_serde_1_3::decode::Error; + /// Deserializes a type from bytes using the `rmp-serde` `1.3` crate. + fn decode(data: Vec) -> Result { + rmp_serde_1_3::decode::from_slice(&data) + } +} diff --git a/libraries/native_model/src/header.rs b/libraries/native_model/src/header.rs new file mode 100644 index 00000000..d2bee226 --- /dev/null +++ b/libraries/native_model/src/header.rs @@ -0,0 +1,9 @@ +use zerocopy::little_endian::U32; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(FromBytes, IntoBytes, Immutable, KnownLayout, Debug)] +#[repr(C)] +pub struct Header { + pub(crate) id: U32, + pub(crate) version: U32, +} diff --git a/libraries/native_model/src/lib.rs b/libraries/native_model/src/lib.rs new file mode 100644 index 00000000..bc49f198 --- /dev/null +++ b/libraries/native_model/src/lib.rs @@ -0,0 +1,184 @@ +//! `native_model` is a Rust crate that acts as a thin wrapper around serialized data, adding identity and version information. +//! +//! - It aims to ensure: +//! - **Interoperability**: Different applications can work together even if they use different data model versions. +//! - **Data Consistency**: Ensures the data is processed as expected. +//! - **Flexibility**: Allows the use of any serialization format. Mode details [here](https://github.com/Drop-OSS/native_model#setup-your-serialization-format). +//! - **Minimal Performance Overhead**: Current performance has a minimal overhead see [performance](https://github.com/Drop-OSS/native_model#performance) section. +//! - **Suitability**: +//! - Suitable for applications that are written in Rust, evolve independently, store data locally, and require incremental upgrades. +//! - Not suitable for non-Rust applications, systems not controlled by the user, or when human-readable formats are needed. +//! - **Setup**: +//! - Users must define their own serialization format and data model. Mode details [here](https://github.com/Drop-OSS/native_model#setup-your-serialization-format). +//! - **Development Stage**: +//! - The crate is in early development, and performance is expected to improve over time. +//! +//! See examples in the [README.md](https://github.com/Drop-OSS/native_model) file. + +#[cfg(doctest)] +#[macro_use] +extern crate doc_comment; + +#[cfg(doctest)] +doc_comment! { + include_str!("../README.md") +} + +#[cfg(any( + feature = "serde", + feature = "bincode_1_3", + feature = "bincode_2", + feature = "postcard_1_0", + feature = "rmp_serde_1_3", + doc +))] +mod codec; + +#[cfg(any( + feature = "serde", + feature = "bincode_1_3", + feature = "bincode_2", + feature = "postcard_1_0", + feature = "rmp_serde_1_3", + doc +))] +pub use codec::*; +mod header; +pub mod wrapper; + +// Macro to generate a [`native_model`] implementation for a struct. +pub use native_model_macro::*; + +use wrapper::*; + +use thiserror::Error; + +pub type Result = std::result::Result; + +#[derive(Error, Debug)] +pub enum Error { + #[error("Invalid header")] + InvalidHeader, + #[error("Failed to decode native model")] + DecodeError, + #[error(transparent)] + DecodeBodyError(#[from] DecodeBodyError), + #[error(transparent)] + EncodeBodyError(#[from] EncodeBodyError), + #[error(transparent)] + UpgradeError(#[from] UpgradeError), + #[error("Upgrade from {} to {} is not supported", from, to)] + UpgradeNotSupported { from: u32, to: u32 }, + #[error(transparent)] + DowngradeError(#[from] DowngradeError), + #[error("Downgrade from {} to {} is not supported", from, to)] + DowngradeNotSupported { from: u32, to: u32 }, + #[error("Wrong type id expected: {}, actual: {}", expected, actual)] + WrongTypeId { expected: u32, actual: u32 }, +} + +pub type DecodeResult = std::result::Result; + +#[derive(Error, Debug)] +#[error("Decode body error: {msg}")] +pub enum DecodeBodyError { + #[error("Mismatched model id")] + MismatchedModelId, + #[error("Decode error: {msg}")] + DecodeError { + msg: String, + #[source] + source: anyhow::Error, + }, +} + +pub type EncodeResult = std::result::Result; + +#[derive(Error, Debug)] +#[error("Encode body error: {msg}")] +pub struct EncodeBodyError { + pub msg: String, + #[source] + pub source: anyhow::Error, +} + +#[derive(Error, Debug)] +#[error("Upgrade error: {msg}")] +pub struct UpgradeError { + pub msg: String, + #[source] + pub source: anyhow::Error, +} + +#[derive(Error, Debug)] +#[error("Downgrade error: {msg}")] +pub struct DowngradeError { + pub msg: String, + #[source] + pub source: anyhow::Error, +} + +/// Allows to encode a [`native_model`] into a [`Vec`]. +/// +/// See examples: +/// - [README.md](https://github.com/Drop-OSS/native_model) file. +/// - other [examples](https://github.com/Drop-OSS/native_model/tree/master/tests/example) +/// +/// # Errors +/// +/// The errors returned from this function depend on the [`Encode`] trait +/// implementor (the serializer), i.e. `bincode_2`. +pub fn encode(model: &T) -> Result> { + T::native_model_encode(model) +} + +/// Allows to decode a [`native_model`] from a [`Vec`] and returns the version ([`u32`]). +/// See examples: +/// - [README.md](https://github.com/Drop-OSS/native_model) file. +/// - other [examples](https://github.com/Drop-OSS/native_model/tree/master/tests/example) +/// +/// # Errors +/// +/// The errors returned from this function depend on the [`Decode`] trait +/// implementor (the deserializer), i.e. `bincode_2`. +pub fn decode(data: Vec) -> Result<(T, u32)> { + T::native_model_decode(data) +} + +pub trait Model: Sized { + fn native_model_id() -> u32; + fn native_model_id_str() -> &'static str; + fn native_model_version() -> u32; + fn native_model_version_str() -> &'static str; + + // --------------- Decode --------------- + fn native_model_decode_body(data: Vec, id: u32) -> DecodeResult; + + fn native_model_decode_upgrade_body(data: Vec, id: u32, version: u32) -> Result; + + fn native_model_decode(data: impl AsRef<[u8]>) -> Result<(Self, u32)> { + let native_model = crate::Wrapper::deserialize(data.as_ref()).unwrap(); + let source_id = native_model.get_id(); + let source_version = native_model.get_version(); + let result = Self::native_model_decode_upgrade_body( + native_model.value().to_vec(), + source_id, + source_version, + )?; + Ok((result, source_version)) + } + + // --------------- Encode --------------- + + fn native_model_encode_body(&self) -> EncodeResult>; + + fn native_model_encode(&self) -> Result> { + let mut data = self.native_model_encode_body()?; + let data = crate::native_model_encode( + &mut data, + Self::native_model_id(), + Self::native_model_version(), + ); + Ok(data) + } +} diff --git a/libraries/native_model/src/wrapper.rs b/libraries/native_model/src/wrapper.rs new file mode 100644 index 00000000..dd42a262 --- /dev/null +++ b/libraries/native_model/src/wrapper.rs @@ -0,0 +1,71 @@ +use crate::header::Header; +use zerocopy::little_endian::U32; +use zerocopy::{IntoBytes, Ref, SplitByteSlice, SplitByteSliceMut}; + +pub struct Wrapper { + header: Ref, + value: T, +} + +impl Wrapper { + pub fn deserialize(packed: T) -> Option { + let (header_lv, rest) = Ref::<_, Header>::from_prefix(packed).ok()?; + let native_model = Self { + header: header_lv, + value: rest, + }; + Some(native_model) + } + + pub const fn value(&self) -> &T { + &self.value + } + + pub fn get_type_id(&self) -> u32 { + self.header.id.get() + } + + pub fn get_id(&self) -> u32 { + self.header.id.get() + } + + pub fn get_version(&self) -> u32 { + self.header.version.get() + } +} + +impl Wrapper { + pub fn set_type_id(&mut self, type_id: u32) { + self.header.id = U32::new(type_id); + } + + pub fn set_version(&mut self, version: u32) { + self.header.version = U32::new(version); + } +} + +pub fn native_model_encode(data: &mut Vec, type_id: u32, version: u32) -> Vec { + let header = Header { + id: U32::new(type_id), + version: U32::new(version), + }; + let mut header = header.as_bytes().to_vec(); + header.append(data); + header +} + +#[cfg(test)] +mod tests { + use crate::{native_model_encode, Wrapper}; + + #[test] + fn native_model_deserialize_with_body() { + let mut data = vec![0u8; 8]; + let data = native_model_encode(&mut data, 200000, 100000); + assert_eq!(data.len(), 16); + let model = Wrapper::deserialize(&data[..]).unwrap(); + assert_eq!(model.get_type_id(), 200000); + assert_eq!(model.get_version(), 100000); + assert_eq!(model.value().len(), 8); + } +} diff --git a/libraries/native_model/tests_crate/.gitignore b/libraries/native_model/tests_crate/.gitignore new file mode 100644 index 00000000..2ebc5ea0 --- /dev/null +++ b/libraries/native_model/tests_crate/.gitignore @@ -0,0 +1,2 @@ +/target +/Cargo.lock \ No newline at end of file diff --git a/libraries/native_model/tests_crate/Cargo.toml b/libraries/native_model/tests_crate/Cargo.toml new file mode 100644 index 00000000..2c381818 --- /dev/null +++ b/libraries/native_model/tests_crate/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "tests_crate" +version = "0.1.0" +edition = "2021" + +[workspace] + +[dependencies] +native_model = { path = "../", no-default-features = true } +serde = { version = "1.0.200", features = ["derive"], optional = true } +bincode = { version = "2.0.0-rc.3", features = ["serde"] , optional = true } +postcard = { version = "1.0.8", features = ["alloc"], optional = true } +anyhow = "1.0.82" + + +[features] +default = ["bincode_1_3"] +bincode_1_3 = ["serde", "native_model/bincode_1_3"] +bincode_2 = ["serde", "native_model/bincode_2", "bincode"] +postcard_1_0 = ["serde", "native_model/postcard_1_0", "postcard"] diff --git a/libraries/native_model/tests_crate/tests/_example.rs b/libraries/native_model/tests_crate/tests/_example.rs new file mode 100644 index 00000000..4fb02733 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/_example.rs @@ -0,0 +1 @@ +mod example; diff --git a/libraries/native_model/tests_crate/tests/example/custom_codec/bincode.rs b/libraries/native_model/tests_crate/tests/example/custom_codec/bincode.rs new file mode 100644 index 00000000..9b1cadff --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/custom_codec/bincode.rs @@ -0,0 +1,34 @@ +use bincode; +use bincode::{config, Decode, Encode}; + +pub struct Bincode; + +impl native_model::Encode for Bincode { + type Error = bincode::error::EncodeError; + fn encode(obj: &T) -> Result, bincode::error::EncodeError> { + bincode::encode_to_vec(obj, config::standard()) + } +} + +impl native_model::Decode for Bincode { + type Error = bincode::error::DecodeError; + fn decode(data: Vec) -> Result { + bincode::decode_from_slice(&data, config::standard()).map(|(result, _)| result) + } +} + +use native_model::native_model; + +#[derive(Encode, Decode, PartialEq, Debug)] +#[native_model(id = 1, version = 1, with = Bincode)] +struct DotV1(u32, u32); + +#[test] +fn test_bincode_encode_decode() { + // Application 1 + let dot = DotV1(1, 2); + let bytes = native_model::encode(&dot).unwrap(); + // Application 1 + let (dot, _) = native_model::decode::(bytes).unwrap(); + assert_eq!(dot, DotV1(1, 2)); +} diff --git a/libraries/native_model/tests_crate/tests/example/custom_codec/bincode_serde.rs b/libraries/native_model/tests_crate/tests/example/custom_codec/bincode_serde.rs new file mode 100644 index 00000000..b2ad9308 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/custom_codec/bincode_serde.rs @@ -0,0 +1,34 @@ +use bincode; +use serde::{Deserialize, Serialize}; + +pub struct Bincode; + +impl native_model::Encode for Bincode { + type Error = bincode::error::EncodeError; + fn encode(obj: &T) -> Result, bincode::error::EncodeError> { + bincode::serde::encode_to_vec(obj, bincode::config::standard()) + } +} + +impl Deserialize<'a>> native_model::Decode for Bincode { + type Error = bincode::error::DecodeError; + fn decode(data: Vec) -> Result { + Ok(bincode::serde::decode_from_slice(&data, bincode::config::standard())?.0) + } +} + +use native_model::native_model; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +#[native_model(id = 1, version = 1, with = Bincode)] +struct DotV1(u32, u32); + +#[test] +fn test_bincode_serde_serialize_deserialize() { + // Application 1 + let dot = DotV1(1, 2); + let bytes = native_model::encode(&dot).unwrap(); + // Application 1 + let (dot, _) = native_model::decode::(bytes).unwrap(); + assert_eq!(dot, DotV1(1, 2)); +} diff --git a/libraries/native_model/tests_crate/tests/example/custom_codec/mod.rs b/libraries/native_model/tests_crate/tests/example/custom_codec/mod.rs new file mode 100644 index 00000000..cd23757b --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/custom_codec/mod.rs @@ -0,0 +1,2 @@ +mod bincode; +mod bincode_serde; diff --git a/libraries/native_model/tests_crate/tests/example/default_codec/bincode_1_3.rs b/libraries/native_model/tests_crate/tests/example/default_codec/bincode_1_3.rs new file mode 100644 index 00000000..0472578e --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/default_codec/bincode_1_3.rs @@ -0,0 +1,19 @@ +#![cfg(feature = "bincode_1_3")] +use native_model::native_model; +use serde::Deserialize; +use serde::Serialize; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +#[native_model(id = 1, version = 1, with = native_model::bincode_1_3::Bincode)] +struct Example { + a: u32, + b: u32, +} + +#[test] +fn encode_decode() { + let example = Example { a: 1, b: 2 }; + let bytes = native_model::encode(&example).unwrap(); + let (example, _) = native_model::decode::(bytes).unwrap(); + assert_eq!(example, Example { a: 1, b: 2 }); +} diff --git a/libraries/native_model/tests_crate/tests/example/default_codec/bincode_2.rs b/libraries/native_model/tests_crate/tests/example/default_codec/bincode_2.rs new file mode 100644 index 00000000..b5add8f6 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/default_codec/bincode_2.rs @@ -0,0 +1,19 @@ +#![cfg(feature = "bincode_2")] +use native_model::{native_model}; +use serde::Deserialize; +use serde::Serialize; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +#[native_model(id = 1, version = 1, with = native_model::bincode_2::Bincode)] +struct Example { + a: u32, + b: u32, +} + +#[test] +fn encode_decode() { + let example = Example { a: 1, b: 2 }; + let bytes = native_model::encode(&example).unwrap(); + let (example, _) = native_model::decode::(bytes).unwrap(); + assert_eq!(example, Example { a: 1, b: 2 }); +} diff --git a/libraries/native_model/tests_crate/tests/example/default_codec/default.rs b/libraries/native_model/tests_crate/tests/example/default_codec/default.rs new file mode 100644 index 00000000..a9d90bce --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/default_codec/default.rs @@ -0,0 +1,19 @@ +#![cfg(feature = "bincode_1_3")] +use native_model::native_model; +use serde::Deserialize; +use serde::Serialize; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +#[native_model(id = 1, version = 1)] +struct Example { + a: u32, + b: u32, +} + +#[test] +fn encode_decode() { + let example = Example { a: 1, b: 2 }; + let bytes = native_model::encode(&example).unwrap(); + let (example, _) = native_model::decode::(bytes).unwrap(); + assert_eq!(example, Example { a: 1, b: 2 }); +} diff --git a/libraries/native_model/tests_crate/tests/example/default_codec/mod.rs b/libraries/native_model/tests_crate/tests/example/default_codec/mod.rs new file mode 100644 index 00000000..3aa01d2f --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/default_codec/mod.rs @@ -0,0 +1,4 @@ +mod default; +mod bincode_1_3; +mod bincode_2; +mod postcard_1_0; \ No newline at end of file diff --git a/libraries/native_model/tests_crate/tests/example/default_codec/postcard_1_0.rs b/libraries/native_model/tests_crate/tests/example/default_codec/postcard_1_0.rs new file mode 100644 index 00000000..114d823f --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/default_codec/postcard_1_0.rs @@ -0,0 +1,20 @@ +#![cfg(feature = "postcard_1_0")] +use native_model::{native_model}; +use serde::Deserialize; +use serde::Serialize; + +#[derive(Serialize)] +#[derive(Deserialize, PartialEq, Debug)] +#[native_model(id = 1, version = 1, with = native_model::postcard_1_0::PostCard)] +struct Example { + a: u32, + b: u32, +} + +#[test] +fn encode_decode() { + let example = Example { a: 1, b: 2 }; + let bytes = native_model::encode(&example).unwrap(); + let (example, _) = native_model::decode::(bytes).unwrap(); + assert_eq!(example, Example { a: 1, b: 2 }); +} diff --git a/libraries/native_model/tests_crate/tests/example/example_define_model.rs b/libraries/native_model/tests_crate/tests/example/example_define_model.rs new file mode 100644 index 00000000..37caef71 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/example_define_model.rs @@ -0,0 +1,111 @@ +#![cfg(feature = "bincode")] +use bincode::{config, Decode, Encode}; +use native_model::native_model; + +pub struct Bincode; + +impl native_model::Encode for Bincode { + type Error = bincode::error::EncodeError; + fn encode(obj: &T) -> Result, bincode::error::EncodeError> { + bincode::encode_to_vec(obj, config::standard()) + } +} + +impl> native_model::Decode for Bincode { + type Error = bincode::error::DecodeError; + fn decode(data: Vec) -> Result { + bincode::decode_from_slice(&data, config::standard()).map(|(result, _)| result) + } +} + +#[derive(Encode, Decode, PartialEq, Debug)] +#[native_model(id = 1, version = 1, with = Bincode)] +struct DotV1(u32, u32); + +#[derive(Encode, Decode, PartialEq, Debug)] +#[native_model(id = 1, version = 2, with = Bincode, from = DotV1)] +struct DotV2 { + name: String, + x: u64, + y: u64, +} + +impl From for DotV2 { + fn from(dot: DotV1) -> Self { + DotV2 { + name: "".to_string(), + x: dot.0 as u64, + y: dot.1 as u64, + } + } +} + +impl From for DotV1 { + fn from(dot: DotV2) -> Self { + DotV1(dot.x as u32, dot.y as u32) + } +} + +#[derive(Encode, Decode, PartialEq, Debug)] +#[native_model(id = 1, version = 3, with = Bincode, try_from = (DotV2, anyhow::Error))] +struct DotV3 { + name: String, + cord: Cord, +} + +#[derive(Encode, Decode, PartialEq, Debug)] +struct Cord { + x: u64, + y: u64, +} + +impl TryFrom for DotV3 { + type Error = anyhow::Error; + + fn try_from(dot: DotV2) -> Result { + Ok(DotV3 { + name: dot.name, + cord: Cord { x: dot.x, y: dot.y }, + }) + } +} + +impl TryFrom for DotV2 { + type Error = anyhow::Error; + + fn try_from(dot: DotV3) -> Result { + Ok(DotV2 { + name: dot.name, + x: dot.cord.x, + y: dot.cord.y, + }) + } +} + +#[test] +fn simple_test() { + let dot = DotV1(1, 2); + let bytes = native_model::encode(&dot).unwrap(); + + let (dot_decoded, _) = native_model::decode::(bytes.clone()).unwrap(); + assert_eq!(dot, dot_decoded); + + let (dot_decoded, _) = native_model::decode::(bytes.clone()).unwrap(); + assert_eq!( + DotV2 { + name: "".to_string(), + x: 1, + y: 2 + }, + dot_decoded + ); + + let (dot_decoded, _) = native_model::decode::(bytes.clone()).unwrap(); + assert_eq!( + DotV3 { + name: "".to_string(), + cord: Cord { x: 1, y: 2 } + }, + dot_decoded + ); +} diff --git a/libraries/native_model/tests_crate/tests/example/example_main.rs b/libraries/native_model/tests_crate/tests/example/example_main.rs new file mode 100644 index 00000000..fb7fa69d --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/example_main.rs @@ -0,0 +1,54 @@ +#![cfg(feature = "bincode_1_3")] +use native_model::native_model; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +#[native_model(id = 1, version = 1)] +struct DotV1(u32, u32); + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +#[native_model(id = 1, version = 2, from = DotV1)] +struct DotV2 { + name: String, + x: u64, + y: u64, +} + +impl From for DotV2 { + fn from(dot: DotV1) -> Self { + DotV2 { + name: "".to_string(), + x: dot.0 as u64, + y: dot.1 as u64, + } + } +} + +impl From for DotV1 { + fn from(dot: DotV2) -> Self { + DotV1(dot.x as u32, dot.y as u32) + } +} + +#[test] +fn run_example() { + // Application 1 + let dot = DotV1(1, 2); + let bytes = native_model::encode(&dot).unwrap(); + + // Application 1 sends bytes to Application 2. + + // Application 2 + let (mut dot, source_version) = native_model::decode::(bytes).unwrap(); + // Use the struct DataV2 which has more fields and a different structure. + dot.name = "Dot".to_string(); + dot.x = 5; + // Encode the dot with the application 1 version in order to be compatible with it. + let bytes = native_model::encode_downgrade(dot, source_version).unwrap(); + + // Application 2 sends bytes to Application 1. + + // Application 1 + let (dot, _) = native_model::decode::(bytes).unwrap(); + assert_eq!(dot, DotV1(5, 2)); +} diff --git a/libraries/native_model/tests_crate/tests/example/mod.rs b/libraries/native_model/tests_crate/tests/example/mod.rs new file mode 100644 index 00000000..20a38eda --- /dev/null +++ b/libraries/native_model/tests_crate/tests/example/mod.rs @@ -0,0 +1,3 @@ +mod default_codec; +mod example_define_model; +mod example_main; diff --git a/libraries/native_model/tests_crate/tests/macro.rs b/libraries/native_model/tests_crate/tests/macro.rs new file mode 100644 index 00000000..4f7f4cc1 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/macro.rs @@ -0,0 +1,46 @@ +#![cfg(feature = "bincode_1_3")] + +use serde::{Deserialize, Serialize}; +use native_model::{native_model, Model}; + +#[derive(Debug, Serialize, Deserialize)] +#[native_model(id = 1, version = 1)] +struct Foo1 { + x: i32, +} + +#[derive(Debug, Serialize, Deserialize)] +#[native_model(id = 1, version = 2, from = Foo1)] +struct Foo2 { + x: i32, +} + +impl From for Foo2 { + fn from(foo1: Foo1) -> Self { + Foo2 { x: foo1.x } + } +} + +impl From for Foo1 { + fn from(foo2: Foo2) -> Self { + Foo1 { x: foo2.x } + } +} + +#[test] +fn get_id_version_int() { + assert_eq!(Foo1::native_model_id(), 1); + assert_eq!(Foo1::native_model_version(), 1); + + assert_eq!(Foo2::native_model_id(), 1); + assert_eq!(Foo2::native_model_version(), 2); +} + +#[test] +fn get_id_version_str() { + assert_eq!(Foo1::native_model_id_str(), "1"); + assert_eq!(Foo1::native_model_version_str(), "1"); + + assert_eq!(Foo2::native_model_id_str(), "1"); + assert_eq!(Foo2::native_model_version_str(), "2"); +} diff --git a/libraries/native_model/tests_crate/tests/macro_decode_decode_upgrade.rs b/libraries/native_model/tests_crate/tests/macro_decode_decode_upgrade.rs new file mode 100644 index 00000000..d39e9022 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/macro_decode_decode_upgrade.rs @@ -0,0 +1,164 @@ +#![cfg(feature = "bincode_1_3")] +use native_model::native_model; +use native_model::Model; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 1, version = 1)] +struct Foo1 { + x: i32, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 1, version = 2, from = Foo1)] +struct Foo2 { + x: String, +} + +impl From for Foo2 { + fn from(foo1: Foo1) -> Self { + Foo2 { + x: foo1.x.to_string(), + } + } +} + +impl From for Foo1 { + fn from(foo2: Foo2) -> Self { + Foo1 { + x: foo2.x.parse::().unwrap(), + } + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 1, version = 3, from = Foo2)] +enum Foo3 { + X(i32), +} + +impl From for Foo3 { + fn from(foo2: Foo2) -> Self { + Foo3::X(foo2.x.parse::().unwrap()) + } +} + +impl From for Foo2 { + fn from(foo3: Foo3) -> Self { + match foo3 { + Foo3::X(x) => Foo2 { x: x.to_string() }, + } + } +} + +#[test] +fn test_decode_foo1_to_foo2() { + let foo1 = Foo1 { x: 100 }; + let foo1_encoded = foo1.native_model_encode_body().unwrap(); + let foo2_decoded = Foo2::native_model_decode_upgrade_body(foo1_encoded, 1, 1).unwrap(); + assert_eq!(foo1.x.to_string(), foo2_decoded.x); +} + +#[test] +fn test_decode_foo2_to_foo3() { + let foo2 = Foo2 { + x: "100".to_string(), + }; + let foo2_encoded = foo2.native_model_encode_body().unwrap(); + let foo3_decoded = Foo3::native_model_decode_upgrade_body(foo2_encoded, 1, 2).unwrap(); + assert_eq!(Foo3::X(100), foo3_decoded); +} + +#[test] +fn test_decode_foo1_to_foo3() { + let foo1 = Foo1 { x: 100 }; + let foo1_encoded = foo1.native_model_encode_body().unwrap(); + let foo3_decoded = Foo3::native_model_decode_upgrade_body(foo1_encoded, 1, 1).unwrap(); + assert_eq!(Foo3::X(100), foo3_decoded); +} + +#[test] +fn test_decode_foo1_to_foo1() { + let foo1 = Foo1 { x: 100 }; + let foo1_encoded = foo1.native_model_encode_body().unwrap(); + let foo1_decoded = Foo1::native_model_decode_upgrade_body(foo1_encoded, 1, 1).unwrap(); + assert_eq!(foo1, foo1_decoded); +} + +#[test] +fn test_decode_foo2_to_foo2() { + let foo2 = Foo2 { + x: "100".to_string(), + }; + let foo2_encoded = foo2.native_model_encode_body().unwrap(); + let foo2_decoded = Foo2::native_model_decode_upgrade_body(foo2_encoded, 1, 2).unwrap(); + assert_eq!(foo2, foo2_decoded); +} + +#[test] +fn test_decode_foo3_to_foo3() { + let foo3 = Foo3::X(100); + let foo3_encoded = foo3.native_model_encode_body().unwrap(); + let foo3_decoded = Foo3::native_model_decode_upgrade_body(foo3_encoded, 1, 3).unwrap(); + assert_eq!(foo3, foo3_decoded); +} + +#[test] +fn test_should_fail_decode_foo3_to_foo2() { + let foo3 = Foo3::X(100); + let foo3_encoded = foo3.native_model_encode_body().unwrap(); + let foo3_decoded = Foo2::native_model_decode_upgrade_body(foo3_encoded, 1, 3); + assert!(foo3_decoded.is_err()); + assert!(matches!( + foo3_decoded.unwrap_err(), + native_model::Error::UpgradeNotSupported { from: 3, to: 2 } + )); +} + +#[test] +fn test_should_fail_decode_foo3_to_foo1() { + let foo3 = Foo3::X(100); + let foo3_encoded = foo3.native_model_encode_body().unwrap(); + let foo3_decoded = Foo1::native_model_decode_upgrade_body(foo3_encoded, 1, 3); + assert!(foo3_decoded.is_err()); + assert!(matches!( + foo3_decoded.unwrap_err(), + native_model::Error::UpgradeNotSupported { from: 3, to: 1 } + )); +} + +#[test] +fn test_should_fail_decode_foo2_to_foo1() { + let foo2 = Foo2 { + x: "100".to_string(), + }; + let foo2_encoded = foo2.native_model_encode_body().unwrap(); + let foo2_decoded = Foo1::native_model_decode_upgrade_body(foo2_encoded, 1, 2); + assert!(foo2_decoded.is_err()); + assert!(matches!( + foo2_decoded.unwrap_err(), + native_model::Error::UpgradeNotSupported { from: 2, to: 1 } + )); +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 2, version = 1)] +struct Foo1Bis { + x: i32, +} + +#[test] +fn test_prevent_to_decode_the_wrong_model() { + let foo1 = Foo1 { x: 100 }; + let foo1_encoded = foo1.native_model_encode_body().unwrap(); + let foo1_decoded = Foo1Bis::native_model_decode_upgrade_body(foo1_encoded, 1, 1); + dbg!(&foo1_decoded); + // assert!(foo1_decoded.is_err()); + // assert!(matches!( + // foo1_decoded.unwrap_err(), + // native_model::Error::TypeIdMismatch { + // expected: 1, + // actual: 1 + // } + // )); +} diff --git a/libraries/native_model/tests_crate/tests/macro_encode_decode.rs b/libraries/native_model/tests_crate/tests/macro_encode_decode.rs new file mode 100644 index 00000000..af2eef10 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/macro_encode_decode.rs @@ -0,0 +1,42 @@ +#![cfg(feature = "bincode_1_3")] + +use native_model::native_model; +use native_model::Model; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 1, version = 1)] +struct Foo1 { + x: i32, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 1, version = 2, from = Foo1)] +struct Foo2 { + x: i32, +} + +impl From for Foo2 { + fn from(foo1: Foo1) -> Self { + Foo2 { x: foo1.x } + } +} + +impl From for Foo1 { + fn from(foo2: Foo2) -> Self { + Foo1 { x: foo2.x } + } +} + +#[test] +fn test_simple() { + let foo1 = Foo1 { x: 100 }; + let foo2 = Foo2 { x: 200 }; + let foo1_encoded = foo1.native_model_encode().unwrap(); + let foo2_encoded = foo2.native_model_encode().unwrap(); + + let (foo1_decoded, _) = Foo1::native_model_decode(foo1_encoded).unwrap(); + assert!(foo1_decoded == foo1); + let (foo2_decoded, _) = Foo2::native_model_decode(foo2_encoded).unwrap(); + assert!(foo2_decoded == foo2); +} diff --git a/libraries/native_model/tests_crate/tests/mod.rs b/libraries/native_model/tests_crate/tests/mod.rs new file mode 100644 index 00000000..4fb02733 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/mod.rs @@ -0,0 +1 @@ +mod example; diff --git a/libraries/native_model/tests_crate/tests/native_model_from.rs b/libraries/native_model/tests_crate/tests/native_model_from.rs new file mode 100644 index 00000000..2c280e76 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/native_model_from.rs @@ -0,0 +1,96 @@ +#![cfg(feature = "bincode_1_3")] + +use native_model::native_model; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 1, version = 1)] +struct Foo1 { + x: i32, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 1, version = 2, from = Foo1)] +struct Foo2 { + x: i32, + c: char, +} + +impl From for Foo2 { + fn from(foo1: Foo1) -> Self { + Foo2 { x: foo1.x, c: 'a' } + } +} + +impl From for Foo1 { + fn from(foo2: Foo2) -> Self { + Foo1 { x: foo2.x } + } +} + +impl PartialEq for Foo2 { + fn eq(&self, other: &Foo1) -> bool { + self.x == other.x + } +} + +#[test] +fn test_decode_foo1_to_foo1() { + let foo1 = Foo1 { x: 100 }; + let foo1_packed = native_model::encode(&foo1).unwrap(); + let (foo1_decoded, _) = native_model::decode::(foo1_packed.clone()).unwrap(); + assert_eq!(foo1, foo1_decoded); +} + +#[test] +fn test_decode_foo1_to_foo2() { + let foo1 = Foo1 { x: 100 }; + let foo1_packed = native_model::encode(&foo1).unwrap(); + let (foo2_decoded, _) = native_model::decode::(foo1_packed.clone()).unwrap(); + assert_eq!(Foo2 { x: 100, c: 'a' }, foo2_decoded); +} + +#[test] +fn test_encode_foo2_to_foo1() { + let foo2 = Foo2 { x: 100, c: 'a' }; + let foo2_packed = native_model::encode(&foo2).unwrap(); + assert_eq!(foo2_packed, vec![1, 0, 0, 0, 2, 0, 0, 0, 100, 0, 0, 0, 97]); + let (foo2_decoded, _) = native_model::decode::(foo2_packed.clone()).unwrap(); + assert_eq!(Foo2 { x: 100, c: 'a' }, foo2_decoded); + let foo1_packed = native_model::encode_downgrade(foo2, 1).unwrap(); + assert_eq!(foo1_packed, vec![1, 0, 0, 0, 1, 0, 0, 0, 100, 0, 0, 0]); + let (foo1_decoded, _) = native_model::decode::(foo1_packed.clone()).unwrap(); + assert_eq!(Foo1 { x: 100 }, foo1_decoded); +} + +#[test] +fn test_encode_foo1_to_foo1() { + let foo1 = Foo1 { x: 100 }; + let foo1_packed = native_model::encode(&foo1).unwrap(); + assert_eq!(foo1_packed, vec![1, 0, 0, 0, 1, 0, 0, 0, 100, 0, 0, 0]); + let (foo1_decoded, _) = native_model::decode::(foo1_packed.clone()).unwrap(); + assert_eq!(Foo1 { x: 100 }, foo1_decoded); + let foo1_packed = native_model::encode_downgrade(foo1, 1).unwrap(); + assert_eq!(foo1_packed, vec![1, 0, 0, 0, 1, 0, 0, 0, 100, 0, 0, 0]); + let (foo1_decoded, _) = native_model::decode::(foo1_packed.clone()).unwrap(); + assert_eq!(Foo1 { x: 100 }, foo1_decoded); +} + +#[test] +fn encode_decode_with_same_version() { + // Client 1 + let foo1 = Foo1 { x: 100 }; + let foo_packed = native_model::encode(&foo1).unwrap(); + // Send foo_packed to server + + // Server + let (mut foo2, version) = native_model::decode::(foo_packed.clone()).unwrap(); + // Do something with foo2 + foo2.x += 1; + let foo_packed = native_model::encode_downgrade(foo2, version).unwrap(); + // Send foo_packed back to client + + // Client + let (foo1_decoded, _) = native_model::decode::(foo_packed.clone()).unwrap(); + assert_eq!(Foo1 { x: 101 }, foo1_decoded); +} diff --git a/libraries/native_model/tests_crate/tests/native_model_try_from.rs b/libraries/native_model/tests_crate/tests/native_model_try_from.rs new file mode 100644 index 00000000..04d05445 --- /dev/null +++ b/libraries/native_model/tests_crate/tests/native_model_try_from.rs @@ -0,0 +1,68 @@ +#![cfg(feature = "bincode_1_3")] + +use native_model::native_model; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 1, version = 1)] +struct Foo1 { + x: i32, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[native_model(id = 1, version = 2, try_from = (Foo1, anyhow::Error))] +struct Foo2 { + x: i32, +} + +impl TryFrom for Foo2 { + type Error = anyhow::Error; + + fn try_from(foo1: Foo1) -> Result { + if foo1.x > 10 { + return Err(anyhow::anyhow!("x > 10")); + } + + Ok(Foo2 { x: foo1.x }) + } +} + +impl TryFrom for Foo1 { + type Error = anyhow::Error; + + fn try_from(foo2: Foo2) -> Result { + if foo2.x > 10 { + return Err(anyhow::anyhow!("x > 10")); + } + + Ok(Foo1 { x: foo2.x }) + } +} + +#[test] +fn test_foo1_to_foo1() { + let foo1 = Foo1 { x: 1 }; + let foo1_packed = native_model::encode(&foo1).unwrap(); + let (foo1_decoded, _) = native_model::decode::(foo1_packed.clone()).unwrap(); + assert_eq!(foo1, foo1_decoded); +} + +#[test] +fn test_foo1_to_foo2() { + let foo1 = Foo1 { x: 1 }; + let foo1_packed = native_model::encode(&foo1).unwrap(); + let (foo2_decoded, _) = native_model::decode::(foo1_packed.clone()).unwrap(); + assert_eq!(Foo2 { x: 1 }, foo2_decoded); +} + +#[test] +fn test_foo1_to_foo2_error() { + let foo1 = Foo1 { x: 1000 }; + let foo1_packed = native_model::encode(&foo1).unwrap(); + let foo2_decoded = native_model::decode::(foo1_packed.clone()); + assert!(foo2_decoded.is_err()); + assert!(matches!( + foo2_decoded.unwrap_err(), + native_model::Error::UpgradeError(_) + )); +} diff --git a/libraries/native_model/version_update.sh b/libraries/native_model/version_update.sh new file mode 100755 index 00000000..95fd074c --- /dev/null +++ b/libraries/native_model/version_update.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Bash script to update version for native_model_macro + +# Semantic release version obtained from argument +NEW_VERSION=$1 + +# Exit if NEW_VERSION is not set +if [ -z "$NEW_VERSION" ]; then + echo "NEW_VERSION argument not set" + exit 1 +fi + +# Directories containing Cargo.toml files to update +declare -a directories=("." "native_model_macro") + +for directory in "${directories[@]}" +do + # Check if Cargo.toml exists in the directory + if [ -f "$directory/Cargo.toml" ]; then + echo "Updating version in $directory/Cargo.toml to $NEW_VERSION" + # Use sed to find and replace the version string + sed -i -E "s/^version = \"[0-9]+\.[0-9]+\.[0-9]+\"/version = \"$NEW_VERSION\"/g" "$directory/Cargo.toml" + + # Update the dependency version for native_model_macro in native_model_macro's Cargo.toml + if [ "$directory" == "." ]; then + sed -i -E "s/native_model_macro = \{ version = \"[0-9]+\.[0-9]+\.[0-9]+\", path = \"native_model_macro\" \}/native_model_macro = { version = \"$NEW_VERSION\", path = \"native_model_macro\" }/g" "$directory/Cargo.toml" + fi + fi +done + + +cd "$DIR/" + +# Commit +git commit --all --message "chore: update version to $NEW_VERSION" +git push \ No newline at end of file