Update readme, update subcomponents, change /inbox to return 202, add Onion-Location to all responses, implement real /api/v1/instance/peers based on known peers, extract Onion-Location from fetches and outgoing federation responses, start migrations from hash_index to trie for account-from-url and status-from-url indexes, change output of /owner/collections/featured so Pleroma doesn't choke on it, remove statuses that have been deleted from timelines when loading

README.md

@@ -9,6 +9,12 @@ functionality one would expect from an ActivityPub server. Software is unstable.
 
 USE AT YOUR OWN RISK.
 
+# Features
+
+* Single User - It's just you.
+* NoSQL - No need to install or maintain a database.
+* ActivityPub federation - follow interesting people on the many instances of the fediverse.
+
 # Dependencies
 * libCURL
 * OpenSSL

src/ap

@@ -1 +1 @@
-Subproject commit 29d82d84b8bf26dc6c502ed9952356f9c393ca1f
+Subproject commit 64797a74e8955aa500deaff1e6883b7349022892

src/controller/inbox.c

@@ -37,7 +37,7 @@ bool route_inbox( struct http_request* req )
 	if( !http_request_route_method( req, "POST" ) ) { return false; }
 
 	if( envelope_create_from_request( req ) ) {
-		http_request_send_headers( req, 200, "text/plain", true );
+		http_request_send_headers( req, 202, "text/plain", true );
 	} else {
 		http_request_send_headers( req, 400, "text/plain", true );
 	}

src/controller/main.c

@@ -5,6 +5,7 @@
 
 // Model
 #include "model/media.h"
+#include "model/server.h"
 
 // Controller
 #include "controller/mastodon_api.h"
@@ -165,6 +166,10 @@ bool route_request( struct http_request* req )
 {
 	http_request_add_header( req, "Carbon-Emissions-Scope-2", "123456.789" );
 
+	char onion_location[512];
+	snprintf( onion_location,512, "http://%s%s", g_server->tor_hidden_service, http_request_get_full_path(req) );
+	http_request_add_header( req, "Onion-Location", onion_location );
+
 	bool inner( struct http_request* req ) {
 		if( http_request_route( req, "/oauth" ) ) {
 			return route_oauth( req );

src/controller/mastodon_api.c

@@ -16,6 +16,7 @@
 #include "model/server.h"
 #include "model/media.h"
 #include "model/marker.h"
+#include "model/peer.h"
 
 // View
 #include "view/api/Attachement.h"
@@ -316,6 +317,28 @@ bool http_request_is_tor_request( struct http_request* req )
 	return false;
 }
 
+bool handle_peers( struct http_request* req )
+{
+	http_request_send_headers( req, 200, "application/json", true );
+	FILE* f = http_request_get_response_body( req );
+	fprintf( f, "[" );
+	bool first = true;
+	for( int id = 1;; ++id ) {
+		struct peer* p = peer_from_id(id);
+		if( p ) {
+			if( !first ) {
+				fprintf( f, "," );
+			}
+			first = false;
+			fprintf( f, "\"%s\"", p->domain );
+		} else {
+			break;
+		}
+	}
+	fprintf( f, "]" );
+	return true;
+}
+
 // route: /api/v1/
 bool route_mastodon_api( struct http_request* req )
 {
@@ -339,13 +362,7 @@ bool route_mastodon_api( struct http_request* req )
 
 	} else if( http_request_route( req, "instance" ) ) {
 		if( http_request_route_term( req, "/peers" ) ) {
-			http_request_send_headers( req, 200, "application/json", true );
-			FILE* f = http_request_get_response_body( req );
-			fprintf( f, "[" );
-				fprintf( f, "\"poa.st\"" );
-				// TODO: print peers
-			fprintf( f, "]" );
-			return true;
+			return handle_peers(req);
 		} else {
 			http_request_send_headers( req, 200, "application/json", true );
 			FILE* f = http_request_get_response_body( req );

src/controller/outbox.c

@@ -15,6 +15,7 @@
 #include "model/activity.h"
 #include "model/outbox_envelope.h"
 #include "model/peer.h"
+#include "model/fetch.h"
 
 // Submodules
 #include "http/url.h"
@@ -94,6 +95,9 @@ static bool process_envelope( struct outbox_envelope* env )
 	url_get_domain( inbox, domain,sizeof(domain) );
 	p = peer_create_from_domain( domain );
 
+	struct fetch_data fd;
+	fd.p = p;
+
 	printf( "Processing outbox/%d.json\n", env->id );
 	printf( "inbox=%s\b", inbox );
 	printf( "account_id=%d\n", env->account_id );
@@ -180,6 +184,7 @@ static bool process_envelope( struct outbox_envelope* env )
 		HTTP_REQ_HEADER, "Content-Type: application/activity+json",
 		HTTP_REQ_POSTDATA, postdata,
 		HTTP_REQ_RESULT_STATUS, &status_code,
+		HTTP_RES_HEADER_CALLBACK, fetch_handle_header, &fd,
 		NULL
 	};
 

src/ffdb

@@ -1 +1 @@
-Subproject commit fee0874bb1517f7da546ca99fd7c3b9497ed54db
+Subproject commit 7e76a21254e772b6bcc955fcb575b5f40a67dbb4

src/http

@@ -1 +1 @@
-Subproject commit 25265c5f0132646a35c2f0bf21963a079ebee348
+Subproject commit 8ff817b844739deb467de6525ab31c43a62e17ad

src/model/account.c

@@ -210,16 +210,35 @@ struct account* account_from_id( int id )
 
 static bool index_uri_to_account_id( const char* uri, int account_id )
 {
+	char id_str[32];
+	snprintf( id_str,32, "%d", account_id );
+
+	ffdb_trie_set( "data/accounts/by-uri", uri, id_str );
 	return hash_index_set( "data/accounts/uri_index/", uri, account_id );
 }
 
 int account_id_from_uri( const char* uri )
 {
+	char* id_str = ffdb_trie_get( "data/accounts/by-uri", uri );
+	if( id_str ) {
+		int id = atoi(id_str);
+		free(id_str);
+		return id;
+	}
+
 	int result = 0;
 	if( !hash_index_get( "data/accounts/uri_index/", uri, &result ) ) {
 		return -1;
 	}
 
+	// Migrate to trie
+	{
+		char id_str[32];
+		snprintf( id_str,32, "%d", result );
+		ffdb_trie_set( "data/accounts/by-uri", uri, id_str );
+		hash_index_remove( "data/accounts/uri_index/", uri );
+	}
+
 	return result;
 }
 static int lookup_account_id_from_uri( const char* uri )
@@ -430,6 +449,13 @@ void account_save( struct account* a )
 	snprintf( filename, 512, "data/accounts/%d.json", a->id );
 
 	json_write_object_layout_to_file( filename, "\t", account_layout, a );
+
+	if( a->account_url ) {
+		char id_str[32];
+		snprintf( id_str,32, "%d", a->id );
+		ffdb_trie_set( "data/accounts/by-uri", a->account_url, id_str );
+		hash_index_remove( "data/accounts/uri_index/", a->account_url );
+	}
 }
 
 

src/model/account/ap_data.c

@@ -298,12 +298,15 @@ struct ap_object* account_ap_featured( struct account* a )
 	struct ap_object* o = activity_new_local_activity();
 	o->type = ap_OrderedCollection;
 	o->published = time(NULL);
-	o->part_of = aformat( "https://%s/owner/collections/featured", g_server->domain );
-	o->id = strdup(o->part_of);
-	o->first.tag = apaot_object;
-	o->first.ptr = account_ap_featured_page( a, 0 );
 	char buffer[512];
+	o->id = aformat( "https://%s/owner/collections/featured", g_server->domain );
 	o->total_items = ffdb_trie_count( format( buffer,512, "data/accounts/%d/timeline/pinned", a->id ) );
+	o->ordered_items.items = malloc(1);
+
+	if( o->total_items > 0 ) {
+		o->first.tag = apaot_object;
+		o->first.ptr = account_ap_featured_page( a, 0 );
+	}
 	return o;
 }
 struct ap_object* account_ap_featured_page( struct account* a, int page )

src/model/fetch.c

@@ -5,9 +5,11 @@
 #include "model/crypto/keys.h"
 #include "model/crypto/http_sign.h"
 #include "model/server.h"
+#include "model/peer.h"
 
 // Submodules
 #include "http/client/client.h"
+#include "http/url.h"
 #include "util/format.h"
 #include "ap/object.h"
 #include "sha256/sha256.h"
@@ -18,20 +20,29 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
+#include <ctype.h>
 
-static size_t handle_header( char* header, size_t size, size_t nitems, void* user )
+size_t fetch_handle_header( char* header, size_t size, size_t nitems, void* user )
 {
 	int bytes = size * nitems;
 	int result = bytes;
 
-	//printf( "? Header: |%.*s|\n", bytes, header );
+	struct fetch_data* fd = user;
+	if( !fd ) {
+		printf( "No fetch data provided, skippig handling.\n" );
+		return result;
+	}
+
+	printf( "? Header: |%.*s|\n", bytes, header );
 
-	if( 0 != strncmp("onion-location: ",header,sizeof("onion-location: ")-1 ) ) {
+	if( 0 != strncmp("onion-location: http://",header,sizeof("onion-location: http://")-1 ) ) {
 		return result;
 	}
 
-	header += sizeof("onion-location: ")-1;
-	bytes  -= sizeof("onion-location: ")-1;
+	header += (sizeof("onion-location: http://")-1);
+	bytes  -= (sizeof("onion-location: http://")-1);
+
+	//printf( "? Header: |%.*s|\n", bytes, header );
 
 	for( int i = 0; i < bytes; ++i ) {
 		if( header[i] == '/' ) {
@@ -41,8 +52,19 @@ static size_t handle_header( char* header, size_t size, size_t nitems, void* use
 	}
 
 	char* onion_host = strndup( header, bytes );
+	int pos = strlen(onion_host) - 1;
+	while( !isalnum(onion_host[pos]) && !( onion_host[pos] == '.' ) ) {
+		onion_host[pos] = '\0';
+		pos -= 1;
+	}
 
 	printf( "+ Onion Host: |%s|\n", onion_host );
+	if( fd->p ) {
+		printf( "Updating peer onion host to %s\n", onion_host );
+		free(fd->p->tor_hidden_service);
+		fd->p->tor_hidden_service = onion_host;
+		onion_host = NULL;
+	}
 
 	free(onion_host);
 
@@ -54,6 +76,16 @@ static bool do_fetch( const char* uri, FILE* result )
 	char user_agent[512];
 	snprintf( user_agent, sizeof(user_agent), "User-Agent: curl (Apogee/0.1; +https://%s/owner/actor)", g_server->domain );
 
+	// Setup fetch data
+	struct fetch_data fd;
+	memset(&fd,0,sizeof(fd));
+	{
+		char host_domain[512];
+		if( url_get_domain( uri, host_domain, sizeof(host_domain) ) ) {
+			fd.p = peer_from_domain(host_domain);
+		}
+	}
+
 	long status_code;
 	const void* request[] = {
 		HTTP_REQ_URL, uri,
@@ -61,13 +93,19 @@ static bool do_fetch( const char* uri, FILE* result )
 		HTTP_REQ_HEADER, user_agent,
 		HTTP_REQ_OUTFILE, result,
 		HTTP_REQ_RESULT_STATUS, &status_code,
-		HTTP_RES_HEADER_CALLBACK, handle_header,
+		HTTP_RES_HEADER_CALLBACK, fetch_handle_header, (void*)&fd,
 		NULL,
 	};
 	printf( "GET %s\n", uri );
 	http_client_do( request );
 	printf( "GET %s -> %ld\n", uri, status_code );
 
+	// Save peer data
+	if( fd.p ) {
+		peer_save( fd.p );
+		peer_free( fd.p );
+	}
+
 	if( status_code != 200 ) {
 		printf( "Retrying request with HTTP Signature header...\n" );
 

src/model/fetch.h

@@ -1,13 +1,20 @@
 #pragma once
 
 #include <stdbool.h>
+#include <stddef.h>
 
 struct ap_object_ptr_or_ref;
 
+struct fetch_data
+{
+	struct peer* p;
+};
+
 bool pull_remote_file( const char* filename, const char* uri );
 bool pull_remote_file_if_older( const char* filename, const char* uri, int seconds );
 
 char* fetch_remote_file_to_cache( const char* uri, int seconds );
+size_t fetch_handle_header( char* header, size_t size, size_t nitems, void* user );
 
 struct ap_object* fetch_ap_object_ref( const char* uri );
 

src/model/gc.c

@@ -77,7 +77,7 @@ void mark_post( struct bitmap* b, int status_id, bool force )
 	status_free(s);
 }
 
-void gc_run()
+static void sweep_posts()
 {
 	int head = fs_list_get( "data/statuses/HEAD" );
 	int tail = fs_list_get( "data/statuses/TAIL" );
@@ -130,7 +130,7 @@ void gc_run()
 			}
 		}
 
-		if( s ) {
+		if( s && s->remote && !s->pinned ) {
 			if( delete ) {
 				status_delete(s);
 			} else {
@@ -147,3 +147,38 @@ void gc_run()
 	fs_list_set( "data/statuses/TAIL", tail );
 }
 
+/*
+static void sweep_timeline( struct timeline* tl )
+{
+	printf( "Sweeping timeline at %s\n", tl->path );
+	struct status* ss[32];
+	int pos = 0;
+	int count = 1;
+	while( count > 0 ) {
+		count = timeline_load_statuses( tl, pos, 32, ss );
+		for( int i = 0; i < count; ++i ) {
+			status_free(ss[i]);
+		}
+		pos += count;
+	}
+}
+
+static void sweep_timelines()
+{
+	for( int i = 0;; i += 1 ) {
+		struct timeline* tl = timeline_from_id(i);
+		if( !tl ) { return; }
+
+		sweep_timeline(tl);
+
+		timeline_free(tl);
+	}
+}
+*/
+
+void gc_run()
+{
+	sweep_posts();
+	//sweep_timelines();
+}
+

src/model/status.c

@@ -217,9 +217,28 @@ struct status* status_from_uri( const char* uri )
 	if( s ) { return s; }
 
 	int id = -1;
+	char* id_str = ffdb_trie_get( "data/statuses/by-uri", uri );
+	if( id_str ) {
+		struct status* s = status_from_id(atoi(id_str));
+		if( s ) {
+			return s;
+		}
+		free(id_str);
+	}
+
 	if( !hash_index_get( "data/statuses/uri", uri, &id ) ) { return NULL; }
 
-	return status_from_id(id);
+	s = status_from_id(id);
+
+	// Convert from hash_index to trie
+	if( s ) {
+		char id_str[32];
+		snprintf( id_str,32, "%d", s->id );
+		ffdb_trie_set( "data/statuses/by-uri", s->url, id_str );
+		hash_index_remove( "data/statuses/uri", uri );
+	}
+
+	return s;
 }
 void status_add_reply( struct status* s, struct status* child )
 {
@@ -511,7 +530,10 @@ struct status* status_fetch_from_uri( const char* uri )
 		s->url = strdup(uri);
 		status_save_new(s);
 
-		hash_index_set( "data/statuses/uri", uri, s->id );
+		//hash_index_set( "data/statuses/uri", uri, s->id );
+		char id_str[32];
+		snprintf( id_str,32, "%d", s->id );
+		ffdb_trie_set( "data/statuses/by-uri", s->url, id_str );
 	}
 
 	if( !status_sync_from_uri(s,uri) ) {
@@ -618,7 +640,10 @@ void status_save( struct status* s )
 
 	// Index the status
 	if( s->url ) {
-		hash_index_set( "data/statuses/uri", s->url, s->id );
+		//hash_index_set( "data/statuses/uri", s->url, s->id );
+		char id_str[32];
+		snprintf( id_str,32, "%d", s->id );
+		ffdb_trie_set( "data/statuses/by-uri", s->url, id_str );
 	}
 
 	if( s->stub ) {

src/model/timeline.c

@@ -51,24 +51,31 @@ int timeline_load_statuses( struct timeline* tl, int offset_from_head, int count
 	struct {
 		char** items;
 		int count;
-	} values;
+	} keys, values;
 	memset(&values,0,sizeof(values));
+	memset(&keys,0,sizeof(keys));
 
-	ffdb_trie_list( tl->path, offset_from_head, count, NULL, &values );
+	ffdb_trie_list( tl->path, offset_from_head, count, &keys, &values );
 	int result_count = 0;
 	for( int i = 0; i < values.count && result_count < count; ++i ) {
 		char* id_str = values.items[i];
 		struct status* s = status_from_id( atoi(id_str) );
 
 		//printf( "found item: %s, s=%p\n", id_str, s );
-		free( id_str );
 
 		if( s ) {
 			result[result_count] = s;
 			result_count += 1;
+		} else {
+			// Drop the item from the timeline if the status no longer exists locally
+			printf( "Unable to load status %s, dropping from timeline %s\n", id_str, tl->path );
+			ffdb_trie_set( tl->path, keys.items[i], NULL );
 		}
+		free(keys.items[i]);
+		free( id_str );
 	}
 	free(values.items);
+	free(keys.items);
 	return result_count;
 }
 static void key_for_post( struct status* s, char* key, int sizeof_key )