riley/puppy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add derive macros to store

Browse source
This commit is contained in:
Riley Apeldoorn 2024-04-24 19:32:42 +02:00
parent c5bd6a127e
commit bcdd5e6059
14 changed files with 378 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Cargo.lock generated
View file

@ -888,6 +888,12 @@ dependencies = [
[[package]]
name = "macro"
version = "0.0.0"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.60",
]
[[package]]
name = "matchit"
@ -1134,6 +1140,8 @@ dependencies = [
name = "puppy"
version = "0.0.0"
dependencies = [
"bincode",
"chrono",
"fetch",
"store",
]
@ -1494,6 +1502,7 @@ dependencies = [
"bincode",
"chrono",
"derive_more",
"either",
"macro",
"rocksdb",
"tempfile",

51
bin/pupctl/src/main.rs
View file

@ -1,31 +1,26 @@
use puppy::{
store::{
self,
alias::Username,
arrow::{FollowRequested, Follows},
mixin::Profile,
Error,
},
model::{schema, Bite, FollowRequest, Follows, Profile, Username},
store::{self, Error},
tl::Post,
Bite, Key, Store,
Key, Store,
};
fn main() -> store::Result<()> {
// Store::nuke(".state")?;
let db = Store::open(".state")?;
let db = Store::open(".state", schema())?;
println!("creating actors");
let riley = get_or_create_actor(&db, "riley")?;
let linen = get_or_create_actor(&db, "linen")?;
if false {
if true {
println!("creating posts");
puppy::create_post(&db, riley, "@linen <3")?;
puppy::create_post(&db, linen, "@riley <3")?;
}
if false {
if true {
println!("making riley follow linen");
if !db.exists::<Follows>((riley, linen))? {
if !db.exists::<Follows>(riley, linen)? {
println!("follow relation does not exist yet");
if !db.exists::<FollowRequested>((riley, linen))? {
if !db.exists::<FollowRequest>(riley, linen)? {
println!("no pending follow request; creating");
puppy::fr::create(&db, riley, linen)?;
} else {
@ -36,44 +31,46 @@ fn main() -> store::Result<()> {
println!("riley already follows linen");
}
}
println!("Posts on the instance:");
println!("\nPosts on the instance:");
for Post {
id,
content,
author,
} in puppy::tl::fetch_all(&db)?
{
let (_, Profile { account_name, .. }) = db.lookup(author)?;
let Profile { account_name, .. } = db.get_mixin(author)?.unwrap();
let content = content.content.unwrap();
println!("- {id} by @{account_name} ({author}):\n{content}",)
}
println!("Linen's followers:");
println!("\nLinen's followers:");
for id in puppy::fr::followers_of(&db, linen)? {
let (_, Profile { account_name, .. }) = db.lookup(id)?;
let Profile { account_name, .. } = db.get_mixin(id)?.unwrap();
println!("- @{account_name} ({id})");
}
println!("Riley's following:");
println!("\nRiley's following:");
for id in puppy::fr::following_of(&db, riley)? {
let (_, Profile { account_name, .. }) = db.lookup(id)?;
let Profile { account_name, .. } = db.get_mixin(id)?.unwrap();
println!("- @{account_name} ({id})");
}
println!("Biting riley");
puppy::bite_actor(&db, linen, riley).unwrap();
for Bite { id, biter, .. } in puppy::bites_on(&db, riley).unwrap() {
let (_, Profile { account_name, .. }) = db.lookup(biter).unwrap();
println!("riley was bitten by @{account_name} at {}", id.timestamp());
if false {
println!("Biting riley");
puppy::bite_actor(&db, linen, riley).unwrap();
for Bite { id, biter, .. } in puppy::bites_on(&db, riley).unwrap() {
let Profile { account_name, .. } = db.get_mixin(biter)?.unwrap();
println!("riley was bitten by @{account_name} at {}", id.timestamp());
}
}
store::OK
}
fn get_or_create_actor(db: &Store, username: &str) -> Result<Key, Error> {
let user = db.translate::<Username>(username);
let user = db.lookup(Username(username.to_string()));
match user {
Ok(key) => {
Ok(Some(key)) => {
println!("found '{username}' ({key})");
Ok(key)
}
Err(Error::Missing) => {
Ok(None) => {
println!("'{username}' doesn't exist yet, creating");
let r = puppy::create_actor(&db, username);
if let Ok(ref key) = r {

6
lib/macro/Cargo.toml
View file

@ -5,3 +5,9 @@ edition = "2021"
[lib]
path = "src/lib.rs"
proc-macro = true
[dependencies]
syn = { version = '2', features = ['full'] }
quote = '*'
proc-macro2 = '*'
heck = '*'

142
lib/macro/src/arrow.rs Normal file
View file

@ -0,0 +1,142 @@
use heck::AsKebabCase;
use proc_macro::TokenStream;
use quote::{quote, ToTokens};
use syn::{parse_macro_input, Data, DeriveInput, Field, Ident};
pub fn arrow(item: TokenStream) -> TokenStream {
let input = parse_macro_input!(item as DeriveInput);
let Data::Struct(structure) = input.data else {
panic!("Only structs are supported as arrows")
};
match structure.fields {
syn::Fields::Named(fields) => from_named(&input.ident, fields),
syn::Fields::Unnamed(f) if f.unnamed.len() == 1 => {
let first = f.unnamed.first().unwrap();
from_newtype(&input.ident, first)
}
_ => panic!(
"Only newtype structs and structs with named fields can have a derived arrow impl"
),
}
}
fn from_named(name: &Ident, fields: syn::FieldsNamed) -> TokenStream {
let (origin, target, identity) = extract_idents(fields);
match identity {
Some(id) => make_multi_arrow(name, origin, target, id),
None => make_basic_arrow(name, origin, target),
}
}
fn make_basic_arrow(name: &Ident, origin: Ident, target: Ident) -> TokenStream {
let spec = gen_spec(name);
TokenStream::from(quote! {
#spec
impl store::arrow::Arrow for #name {}
impl From<store::arrow::Basic> for #name {
fn from(v: store::arrow::Basic) -> #name {
#name {
#origin: v.origin,
#target: v.target,
}
}
}
impl From<#name> for store::arrow::Basic {
fn from(v: #name) -> store::arrow::Basic {
store::arrow::Basic {
origin: v.#origin,
target: v.#target,
}
}
}
})
}
fn make_multi_arrow(name: &Ident, origin: Ident, target: Ident, id: Ident) -> TokenStream {
let spec = gen_spec(name);
TokenStream::from(quote! {
#spec
impl store::arrow::Arrow for #name {
type Kind = store::arrow::Multi;
}
impl From<store::arrow::Multi> for #name {
fn from(v: store::arrow::Multi) -> #name {
#name {
#id: v.identity,
#origin: v.origin,
#target: v.target,
}
}
}
impl From<#name> for store::arrow::Multi {
fn from(v: #name) -> store::arrow::Multi {
store::arrow::Multi {
identity: v.#id,
origin: v.#origin,
target: v.#target,
}
}
}
})
}
fn extract_idents(fields: syn::FieldsNamed) -> (Ident, Ident, Option<Ident>) {
let origin = extract_ident("origin", &fields).unwrap();
let target = extract_ident("target", &fields).unwrap();
let id = extract_ident("identity", &fields);
(origin, target, id)
}
fn extract_ident(name: &str, fields: &syn::FieldsNamed) -> Option<Ident> {
// Prefer marked fields and default to correctly named fields.
fields
.named
.iter()
.find(|field| {
field
.attrs
.iter()
.filter_map(|attr| attr.meta.path().get_ident())
.any(|id| id == name)
})
.and_then(|f| f.ident.clone())
.or_else(|| {
fields
.named
.iter()
.filter_map(|f| f.ident.clone())
.find(|id| id == name)
})
}
fn gen_spec(name: &Ident) -> impl ToTokens {
let prefix = AsKebabCase(name.to_string());
let by_origin = format!("{prefix}/by-origin");
let by_target = format!("{prefix}/by-target");
quote! {
impl store::types::Value for #name {
type Type = store::types::ArrowSpec;
const SPEC: Self::Type = store::types::ArrowSpec {
by_origin: store::types::Namespace(#by_origin),
by_target: store::types::Namespace(#by_target),
};
}
}
}
fn from_newtype(name: &Ident, field: &Field) -> TokenStream {
let spec = gen_spec(name);
let typ = &field.ty;
TokenStream::from(quote! {
#spec
impl store::arrow::Arrow for #name {
type Kind = #typ;
}
impl From<#typ> for #name {
fn from(v: #typ) -> #name { #name(v) }
}
impl From<#name> for #typ {
fn from(v: #name) -> #typ { v.0 }
}
})
}

63
lib/macro/src/lib.rs
View file

@ -1,16 +1,73 @@
use proc_macro::TokenStream;
mod arrow;
#[proc_macro_derive(Arrow, attributes(origin, target, identity))]
pub fn arrow(item: TokenStream) -> TokenStream {
TokenStream::new()
arrow::arrow(item)
}
#[proc_macro_derive(Alias)]
pub fn alias(item: TokenStream) -> TokenStream {
TokenStream::new()
let input = syn::parse_macro_input!(item as syn::DeriveInput);
let syn::Data::Struct(structure) = input.data else {
panic!("Only structs are supported as aliases")
};
match structure.fields {
syn::Fields::Unnamed(f) if f.unnamed.len() == 1 => {
let first = f.unnamed.first().unwrap();
make_alias_impl(&input.ident, first)
}
_ => panic!("Only string newtype structs are allowed as aliases"),
}
}
fn make_alias_impl(name: &syn::Ident, field: &syn::Field) -> TokenStream {
let typ = &field.ty;
let prefix = heck::AsKebabCase(name.to_string());
let keyspace = format!("{prefix}/keyspace");
let reversed = format!("{prefix}/reversed");
let spec = quote::quote! {
impl store::types::Value for #name {
type Type = store::types::AliasSpec;
const SPEC: Self::Type = store::types::AliasSpec {
keyspace: store::types::Namespace(#keyspace),
reversed: store::types::Namespace(#reversed),
};
}
};
TokenStream::from(quote::quote! {
#spec
impl store::Alias for #name {}
impl AsRef<str> for #name {
fn as_ref(&self) -> &str { self.0.as_ref() }
}
impl From<#typ> for #name {
fn from(v: #typ) -> #name { #name(v) }
}
})
}
#[proc_macro_derive(Mixin)]
pub fn mixin(item: TokenStream) -> TokenStream {
TokenStream::new()
let input = syn::parse_macro_input!(item as syn::DeriveInput);
let name = input.ident;
let prefix = heck::AsKebabCase(name.to_string());
let keyspace = format!("{prefix}/main");
let spec = quote::quote! {
impl store::types::Value for #name {
type Type = store::types::MixinSpec;
const SPEC: Self::Type = store::types::MixinSpec {
keyspace: store::types::Namespace(#keyspace),
};
}
};
TokenStream::from(quote::quote! {
#spec
impl store::Mixin for #name {}
})
}

2
lib/puppy/Cargo.toml
View file

@ -8,3 +8,5 @@ path = "src/lib.rs"
[dependencies]
store = { path = "../store" }
fetch = { path = "../fetch" }
bincode = "2.0.0-rc.3"
chrono = "*"

73
lib/puppy/src/lib.rs
View file

@ -1,10 +1,13 @@
#![feature(iterator_try_collect)]
use model::{AuthorOf, Bite, Content, Profile, Username};
use store::util::{key, IterExt as _};
pub use store::{self, Key, Store};
pub mod model {
use bincode::{Decode, Encode};
use store::{types::Schema, Key};
#[derive(store::Mixin)]
#[derive(store::Mixin, Encode, Decode)]
pub struct Profile {
pub post_count: usize,
pub account_name: String,
@ -13,13 +16,13 @@ pub mod model {
pub about_fields: Vec<(String, String)>,
}
#[derive(store::Mixin)]
#[derive(store::Mixin, Encode, Decode)]
pub struct Content {
pub content: Option<String>,
pub summary: Option<String>,
}
#[derive(store::Arrow, Clone, Copy)]
#[derive(store::Arrow, Debug, PartialEq, Eq, Clone, Copy)]
pub struct AuthorOf {
#[origin]
pub author: Key,
@ -27,7 +30,7 @@ pub mod model {
pub object: Key,
}
#[derive(store::Arrow, Clone, Copy)]
#[derive(store::Arrow, Debug, PartialEq, Eq, Clone, Copy)]
pub struct Follows {
#[origin]
pub follower: Key,
@ -35,7 +38,7 @@ pub mod model {
pub followed: Key,
}
#[derive(store::Arrow, Clone, Copy)]
#[derive(store::Arrow, Debug, PartialEq, Eq, Clone, Copy)]
pub struct Bite {
#[identity]
pub id: Key,
@ -45,7 +48,7 @@ pub mod model {
pub victim: Key,
}
#[derive(store::Arrow, Clone, Copy)]
#[derive(store::Arrow, Debug, PartialEq, Eq, Clone, Copy)]
pub struct FollowRequest {
#[identity]
pub id: Key,
@ -109,7 +112,11 @@ pub fn create_actor(db: &Store, username: impl ToString) -> store::Result<Key> {
pub fn list_posts_by_author(db: &Store, author: Key) -> store::Result<Vec<(Key, Content)>> {
db.run(|tx| {
tx.outgoing::<AuthorOf>(author)
.bind_results(|arr| tx.get_mixin::<Content>(arr.object))
.map_ok(|a| a.object)
.filter_bind_results(|post| {
let thing = tx.get_mixin(post)?;
Ok(thing.map(key(post)))
})
.collect()
})
}
@ -129,7 +136,9 @@ pub fn bites_on(db: &Store, victim: Key) -> store::Result<Vec<Bite>> {
pub mod tl {
//! Timelines
use store::{arrow::AuthorOf, mixin::Content, util::IterExt as _, Error, Key, Result, Store};
use store::{util::IterExt as _, Error, Key, Result, Store};
use crate::model::{AuthorOf, Content};
pub struct Post {
pub id: Key,
@ -139,9 +148,10 @@ pub mod tl {
pub fn fetch_all(db: &Store) -> Result<Vec<Post>> {
db.run(|tx| {
let iter = tx.list::<Content>();
let iter = tx.range::<Content>(..);
iter.bind_results(|(id, content)| {
let author = tx.incoming::<AuthorOf>(id).next_or(Error::Missing)?;
let AuthorOf { author, .. } =
tx.incoming::<AuthorOf>(id).next_or(Error::Missing)?;
Ok(Post {
id,
author,
@ -185,31 +195,35 @@ pub mod fr {
pub fn reject(db: &Store, requester: Key, target: Key) -> store::Result<()> {
db.run(|tx| {
tx.remove_arrow::<FollowRequested>((requester, target))?;
tx.delete_all::<FollowRequest>(requester, target)?;
OK
})
}
pub fn list_pending(db: &Store, target: Key) -> store::Result<Vec<Key>> {
db.transaction(|tx| tx.list_incoming::<FollowRequested>(target).keys().collect())
pub fn list_pending(db: &Store, target: Key) -> store::Result<Vec<FollowRequest>> {
db.incoming::<FollowRequest>(target).collect()
}
pub fn following_of(db: &Store, actor: Key) -> store::Result<Vec<Key>> {
db.transaction(|tx| tx.list_outgoing::<Follows>(actor).keys().collect())
db.outgoing::<Follows>(actor)
.map_ok(|a| a.followed)
.collect()
}
pub fn followers_of(db: &Store, actor: Key) -> store::Result<Vec<Key>> {
db.transaction(|tx| tx.list_incoming::<Follows>(actor).keys().collect())
db.incoming::<Follows>(actor)
.map_ok(|a| a.follower)
.collect()
}
#[cfg(test)]
mod tests {
use store::{
arrow::{FollowRequested, Follows},
Key, Store, OK,
};
use store::{Key, Store, OK};
use crate::create_actor;
use crate::{
create_actor,
model::{schema, FollowRequest, Follows},
};
fn make_test_actors(db: &Store) -> store::Result<(Key, Key)> {
let alice = create_actor(&db, "alice")?;
@ -220,18 +234,21 @@ pub mod fr {
#[test]
fn create_fr() -> store::Result<()> {
Store::with_tmp(|db| {
Store::test(schema(), |db| {
let (alice, bob) = make_test_actors(&db)?;
super::create(&db, alice, bob)?;
assert!(
db.exists::<FollowRequested>((alice, bob))?,
db.exists::<FollowRequest>(alice, bob)?,
"(alice -> bob) ∈ follow-requested"
);
assert!(
!db.exists::<Follows>((alice, bob))?,
!db.exists::<Follows>(alice, bob)?,
"(alice -> bob) ∉ follows"
);
let pending_for_bob = super::list_pending(&db, bob)?;
let pending_for_bob = super::list_pending(&db, bob)?
.into_iter()
.map(|fr| fr.origin)
.collect::<Vec<_>>();
assert_eq!(pending_for_bob, vec![alice], "bob.pending = {{alice}}");
OK
})
@ -239,17 +256,17 @@ pub mod fr {
#[test]
fn accept_fr() -> store::Result<()> {
Store::with_tmp(|db| {
Store::test(schema(), |db| {
let (alice, bob) = make_test_actors(&db)?;
super::create(&db, alice, bob)?;
super::accept(&db, alice, bob)?;
assert!(
db.exists::<Follows>((alice, bob))?,
db.exists::<Follows>(alice, bob)?,
"(alice -> bob) ∈ follows"
);
assert!(
!db.exists::<Follows>((bob, alice))?,
!db.exists::<Follows>(bob, alice)?,
"(bob -> alice) ∉ follows"
);
@ -265,7 +282,7 @@ pub mod fr {
#[test]
fn listing_follow_relations() -> store::Result<()> {
Store::with_tmp(|db| {
Store::test(schema(), |db| {
let (alice, bob) = make_test_actors(&db)?;
super::create(&db, alice, bob)?;
super::accept(&db, alice, bob)?;

1
lib/store/Cargo.toml
View file

@ -13,3 +13,4 @@ bincode = "2.0.0-rc.3"
chrono = "*"
tempfile = "*"
macro = { path = "../macro" }
either = "*"

14
lib/store/src/arrow.rs
View file

@ -93,20 +93,18 @@ impl Store {
op::exists::<A>(self, origin, target)
}
/// Get all arrows of type `A` that point at `target`.
pub fn incoming<'a, A>(&'a self, target: Key) -> impl Iterator<Item = Result<A::Kind>> + 'a
pub fn incoming<'a, A>(&'a self, target: Key) -> impl Iterator<Item = Result<A>> + 'a
where
A::Kind: 'a,
A: Arrow,
A: Arrow + 'a,
{
op::incoming::<A>(self, target)
op::incoming::<A>(self, target).map_ok(A::from)
}
/// Get all arrows of type `A` that point away from `origin`.
pub fn outgoing<'a, A>(&'a self, origin: Key) -> impl Iterator<Item = Result<A::Kind>> + 'a
pub fn outgoing<'a, A>(&'a self, origin: Key) -> impl Iterator<Item = Result<A>> + 'a
where
A::Kind: 'a,
A: Arrow,
A: Arrow + 'a,
{
op::outgoing::<A>(self, origin)
op::outgoing::<A>(self, origin).map_ok(A::from)
}
}

20
lib/store/src/internal.rs
View file

@ -49,6 +49,26 @@ where
.full_iterator(&self.cf, IteratorMode::Start)
.map_err(Error::Internal)
}
/// Execute a range scan
pub fn range<const N: usize>(
&self,
lower: [u8; N],
upper: [u8; N],
) -> impl Iterator<Item = Result<(Box<[u8]>, Box<[u8]>)>> + 'db {
self.context
.full_iterator(&self.cf, IteratorMode::Start)
.skip_while(move |r| match r {
Ok((ref k, _)) => k.as_ref() < &lower,
_ => false,
})
// The prefix iterator may "overshoot". This makes it stop when it reaches
// the end of the range that has the prefix.
.take_while(move |r| match r {
Ok((ref k, _)) => k.as_ref() < &upper,
_ => true,
})
.map_err(Error::Internal)
}
}
impl<C> Keyspace<'_, C>

5
lib/store/src/key.rs
View file

@ -44,6 +44,11 @@ impl Key {
let head = Key::from_slice(&buf[16..]);
(tail, head)
}
pub(crate) fn range(ts: DateTime<Utc>) -> ([u8; 16], [u8; 16]) {
let min = Ulid::from_parts(ts.timestamp_millis() as u64, u128::MIN).to_bytes();
let max = Ulid::from_parts(ts.timestamp_millis() as u64, u128::MAX).to_bytes();
(min, max)
}
fn to_ulid(self) -> Ulid {
Ulid::from_bytes(self.0)
}

3
lib/store/src/lib.rs
View file

@ -1,5 +1,4 @@
#![feature(iterator_try_collect, associated_type_defaults)]
#![feature(marker_trait_attr)]
//! Data persistence for the ActivityPuppy social media server built on top of [rocksdb].
//!
//! # Overview
@ -158,4 +157,4 @@ pub enum Error {
Decoding(bincode::error::DecodeError),
}
pub type Backend = rocksdb::TransactionDB<rocksdb::MultiThreaded>;
type Backend = rocksdb::TransactionDB<rocksdb::MultiThreaded>;

45
lib/store/src/mixin.rs
View file

@ -1,4 +1,11 @@
use std::{
fmt::Pointer,
ops::{Bound, RangeBounds},
};
use bincode::{Decode, Encode};
use chrono::{DateTime, TimeDelta, Utc};
use either::Either;
/// Derive a [`Mixin`] implementation.
pub use r#macro::Mixin;
@ -6,7 +13,7 @@ use super::{
types::{MixinSpec, Value},
Batch, Store, Transaction,
};
use crate::{Error, Key, Result};
use crate::{internal::Query, util::IterExt, Error, Key, Result};
/// Mixins are the simplest pieces of data in the store.
pub trait Mixin: Value<Type = MixinSpec> + Encode + Decode {}
@ -73,6 +80,40 @@ impl Transaction<'_> {
{
op::has_mixin::<M>(self, node)
}
/// Get all `M`s where the key's timestamp is within the `range`.
pub fn range<M>(
&self,
range: impl RangeBounds<DateTime<Utc>>,
) -> impl Iterator<Item = Result<(Key, M)>> + '_
where
M: Mixin,
{
use crate::internal::Context as _;
const MS: TimeDelta = TimeDelta::milliseconds(1);
let iter = match (range.start_bound(), range.end_bound()) {
(Bound::Unbounded, Bound::Unbounded) => {
Either::Left(self.open(M::SPEC.keyspace).list())
}
(min, max) => {
let lower = match min {
Bound::Unbounded => [u8::MIN; 16],
Bound::Included(inc) => Key::range(*inc).0,
Bound::Excluded(exc) => Key::range(*exc + MS).0,
};
let upper = match max {
Bound::Unbounded => [u8::MAX; 16],
Bound::Included(inc) => Key::range(*inc).1,
Bound::Excluded(exc) => Key::range(*exc - MS).1,
};
Either::Right(self.open(M::SPEC.keyspace).range(lower, upper))
}
};
iter.bind_results(|(k, v)| {
let key = Key::from_slice(k.as_ref());
let val = op::decode(v)?;
Ok((key, val))
})
}
}
impl Batch {
@ -136,7 +177,7 @@ mod op {
bincode::encode_to_vec(data, bincode::config::standard()).map_err(Error::Encoding)
}
fn decode<T>(data: impl AsRef<[u8]>) -> Result<T>
pub(super) fn decode<T>(data: impl AsRef<[u8]>) -> Result<T>
where
T: bincode::Decode,
{

14
lib/store/src/util.rs
View file

@ -46,6 +46,20 @@ pub trait IterExt: Iterator + Sized {
{
self.next().ok_or(e)?
}
/// `filter_map` meets `and_then`.
fn filter_bind_results<'a, I, O, E>(
self,
mut f: impl FnMut(I) -> Result<Option<O>, E> + 'a,
) -> impl Iterator<Item = Result<O, E>> + 'a
where
Self: Iterator<Item = Result<I, E>> + 'a,
{
self.filter_map(move |r| r.and_then(|x| f(x)).transpose())
}
}
impl<I> IterExt for I where I: Iterator {}
pub fn key<K, V>(key: K) -> impl FnOnce(V) -> (K, V) {
move |val| (key, val)
}