Fix typo in README, update submodules, implement delivery of activities via TOR, make sure system statuses have unique IDs, fix HTTP signature generation to work with http:// URLs

README.md

@@ -1,4 +1,4 @@
-# apoge
+# apogee
 
 Personal ActivityPub server intended for single-user instances.
 

src/controller/fetch.c

@@ -29,11 +29,11 @@ static void process_fetch_once()
 		char* key = NULL;
 		char* value = NULL;
 		if( !ffdb_trie_get_index( STUBS, idx, &key, &value ) ) {
-			printf( "Unable to get STUBS[%d], skipping\n", idx );
+			//printf( "Unable to get STUBS[%d], skipping\n", idx );
 			continue;
 		}
 
-		printf( "STUBS[%d]: %s = %s\n", idx, key, value );
+		//printf( "STUBS[%d]: %s = %s\n", idx, key, value );
 		struct status* s = status_from_id( atoi(key) );
 		if( !s ) {
 			printf( "Unable to load status with id %s\n", key );

src/controller/outbox.c

@@ -19,6 +19,7 @@
 
 // Submodules
 #include "http/url.h"
+#include "util/format.h"
 
 // Stdlib
 #include <stdlib.h>
@@ -67,6 +68,102 @@ cleanup:
 	return result;
 }
 
+static long submit_activity( const char* postdata, const char* inbox, const char* deliver, struct crypto_keys* keys, struct fetch_data* fd, bool use_tor_proxy )
+{
+	printf( "Using inbox=%s\n", inbox );
+	printf( "Using deliver=%s\n", deliver );
+	printf( "Using proxy=%c\n", use_tor_proxy ? 'Y' : 'N' );
+
+	struct http_signature hs;
+	memset( &hs, 0, sizeof(hs) );
+	hs.input.url = deliver;
+	hs.input.method = "post";
+	hs.input.post_data = postdata;
+	if( !http_signature_make( keys, &hs ) ) {
+		printf( "! Failed to make HTTP signature\n" );
+		return 600;
+	}
+
+	char date_header[512];
+	snprintf( date_header, sizeof(date_header), "Date: %s", hs.date );
+
+	char sign_header[512];
+	snprintf( sign_header, sizeof(sign_header), "Signature: keyId=\"https://%s/owner/actor#mainKey\",headers=\"(request-target) host date content-length digest\",signature=\"%s\"",
+		g_server->domain,
+		hs.signature
+	);
+
+	char user_agent[512];
+	snprintf( user_agent, sizeof(user_agent), "User-Agent: curl (Apogee/0.1; +https://%s/owner/actor)", g_server->domain );
+
+	char digest_header[512];
+	snprintf( digest_header, sizeof(digest_header), "Digest: %s", hs.digest );
+
+	char content_length_header[512];
+	snprintf( content_length_header, sizeof(content_length_header), "Content-Length: %d", hs.content_length );
+
+	char proxy[512];
+	snprintf( proxy,512, "socks5h://localhost:%d", g_server->tor_socks_port );
+
+	printf( "Performing post to %s\n", deliver );
+	long status_code = -1;
+	const void* request_clearnet[] = {
+		HTTP_REQ_URL, deliver,
+		HTTP_REQ_HEADER, user_agent,
+		HTTP_REQ_HEADER, date_header,
+		HTTP_REQ_HEADER, sign_header,
+		HTTP_REQ_HEADER, content_length_header,
+		HTTP_REQ_HEADER, digest_header,
+		HTTP_REQ_HEADER, "Content-Type: application/activity+json",
+		HTTP_REQ_POSTDATA, postdata,
+		HTTP_REQ_RESULT_STATUS, &status_code,
+		HTTP_RES_HEADER_CALLBACK, fetch_handle_header, fd,
+		//HTTP_REQ_PROXY, proxy,
+		NULL
+	};
+
+	const void* request_tor[] = {
+		HTTP_REQ_URL, deliver,
+		HTTP_REQ_HEADER, user_agent,
+		HTTP_REQ_HEADER, date_header,
+		HTTP_REQ_HEADER, sign_header,
+		HTTP_REQ_HEADER, content_length_header,
+		HTTP_REQ_HEADER, digest_header,
+		HTTP_REQ_HEADER, "Content-Type: application/activity+json",
+		HTTP_REQ_POSTDATA, postdata,
+		HTTP_REQ_RESULT_STATUS, &status_code,
+		HTTP_RES_HEADER_CALLBACK, fetch_handle_header, fd,
+		HTTP_REQ_PROXY, proxy,
+		HTTP_REQ_TIMEOUT, (void*)10,
+		NULL
+	};
+
+	// POST to inbox
+	const void** request = request_clearnet;
+	if( use_tor_proxy ) {
+		request = request_tor;
+	}
+
+	printf( "\n\nRequest result:\n" );
+	fflush(stdout);
+	http_client_do(request);
+	printf( "\n\n" );
+	fflush(stdout);
+
+	http_signature_free( &hs );
+
+	return status_code;
+}
+static bool delivery_succeeded( long status_code )
+{
+	switch( status_code ) {
+		case 200:
+		case 202:
+			return true;
+	}
+	return false;
+}
+
 static bool process_envelope( struct outbox_envelope* env )
 {
 	bool result = false;
@@ -76,6 +173,7 @@ static bool process_envelope( struct outbox_envelope* env )
 	struct ap_object* act = NULL;
 	struct account* to_account = NULL;
 	struct peer* p = NULL;
+	char* tor_inbox = NULL;
 
 	if( env->sent ) { return false; }
 	if( env->retry_after > time(NULL) ) { return false; }
@@ -99,6 +197,15 @@ static bool process_envelope( struct outbox_envelope* env )
 	url_get_domain( inbox, domain,sizeof(domain) );
 	p = peer_create_from_domain( domain );
 
+	// Handle tor delivery
+	//*
+	if( p->tor_hidden_service ) {
+		char* path = strchr( inbox + 9, '/' );
+		tor_inbox = aformat( "http://%s%s", p->tor_hidden_service, path );
+		printf( "Using TOR inbox: %s\n", tor_inbox );
+	}
+	//*/
+
 	struct fetch_data fd;
 	fd.p = p;
 
@@ -148,87 +255,62 @@ static bool process_envelope( struct outbox_envelope* env )
 	postdata[size] = '\0';
 	printf( "post: %s\n", postdata );
 
-	struct http_signature hs;
-	memset( &hs, 0, sizeof(hs) );
-	hs.input.url = inbox;
-	hs.input.method = "post";
-	hs.input.post_data = postdata;
-	if( !http_signature_make( keys, &hs ) ) {
-		printf( "! Failed to make HTTP signature\n" );
-		goto failed;
-	}
-
-	char date_header[512];
-	snprintf( date_header, sizeof(date_header), "Date: %s", hs.date );
-
-	char sign_header[512];
-	snprintf( sign_header, sizeof(sign_header), "Signature: keyId=\"https://%s/owner/actor#mainKey\",headers=\"(request-target) host date content-length digest\",signature=\"%s\"",
-		g_server->domain,
-		hs.signature
-	);
+	long status_code;
 
-	char user_agent[512];
-	snprintf( user_agent, sizeof(user_agent), "User-Agent: curl (Apogee/0.1; +https://%s/owner/actor)", g_server->domain );
-
-	char digest_header[512];
-	snprintf( digest_header, sizeof(digest_header), "Digest: %s", hs.digest );
-
-	char content_length_header[512];
-	snprintf( content_length_header, sizeof(content_length_header), "Content-Length: %d", hs.content_length );
+	// Try TOR Hidden Service delivery
+	if( tor_inbox ) {
+		status_code = submit_activity( postdata, inbox, tor_inbox, keys, &fd, true );
+		if( delivery_succeeded(status_code) ) {
+			p->last_hidden_service_delivery = time(NULL);
+			goto success;
+		}
+	}
 
-	printf( "Performing post to %s\n", inbox );
-	long status_code = -1;
-	const void* request[] = {
-		HTTP_REQ_URL, inbox,
-		HTTP_REQ_HEADER, user_agent,
-		HTTP_REQ_HEADER, date_header,
-		HTTP_REQ_HEADER, sign_header,
-		HTTP_REQ_HEADER, content_length_header,
-		HTTP_REQ_HEADER, digest_header,
-		HTTP_REQ_HEADER, "Content-Type: application/activity+json",
-		HTTP_REQ_POSTDATA, postdata,
-		HTTP_REQ_RESULT_STATUS, &status_code,
-		HTTP_RES_HEADER_CALLBACK, fetch_handle_header, &fd,
-		NULL
-	};
+	// Try TOR delivery
+	status_code = submit_activity( postdata, inbox, inbox, keys, &fd, true );
+	if( delivery_succeeded(status_code) ) {
+		p->last_tor_delivery = time(NULL);
+		goto success;
+	}
 
-	// POST to inbox
-	printf( "\n\nRequest result:\n" );
-	fflush(stdout);
-	http_client_do(request);
-	printf( "\n\n" );
-	fflush(stdout);
+	// Try clearnet delivery
+	status_code = submit_activity( postdata, inbox, inbox, keys, &fd, false );
 
-	http_signature_free( &hs );
+	// delivery failed
+	printf( "\nServer returned status code %ld\n", status_code );
+	if( env->retry_after ) {
+		env->retries += 1;
+	}
+	if( p ) {
+		p->last_failed_delivery = time(NULL);
 
-	if( status_code == 200 || status_code == 202 ) {
-		printf( "Submitted successfully\n" );
-		if( p ) {
-			p->last_successful_delivery = time(NULL);
-			peer_save(p);
+		if( p->last_successful_delivery - p->last_failed_delivery > 60*60*24*7 ) {
+			// At least one week since last successful delivery
+			printf( "Over a week since last successful delivery, marking for discard.\n" );
+			env->retries = g_server->outbox_discard_limit;
 		}
+
+		peer_save(p);
+	}
+	if( env->retries >= g_server->outbox_discard_limit ) {
+		// Force discard after 10 delivery attempts
+		// TODO: change this to a configuration option
 		goto discard;
-	} else {
-		printf( "\nServer returned status code %ld\n", status_code );
-		if( env->retry_after ) {
-			env->retries += 1;
-		}
-		if( env->retries > 10 ) {
-			// Force discard after 10 delivery attempts
-			// TODO: change this to a configuration option
-			goto discard;
-		}
-		env->retry_after = time(NULL) + 60 * ( env->retries + 1 ) * ( env->retries + 1 );
-		outbox_envelope_save(env);
-		if( p ) {
-			p->last_failed_delivery = time(NULL);
-			peer_save(p);
-		}
 	}
+	env->retry_after = time(NULL) + 60 * ( env->retries + 1 ) * ( env->retries + 1 );
+	outbox_envelope_save(env);
 
 	goto failed;
 
+success:
+	printf( "Submitted successfully\n" );
+	if( p ) {
+		p->last_successful_delivery = time(NULL);
+		peer_save(p);
+	}
+	goto discard;
 cleanup:
+	free(tor_inbox);
 	ap_object_free(act);
 	account_free(to_account);
 	free(postdata);
@@ -242,6 +324,7 @@ failed:
 	result = false;
 	goto cleanup;
 discard:
+	printf( "Discarding.\n" );
 	env->sent = true;
 	outbox_envelope_save(env);
 	result = true;
@@ -253,24 +336,35 @@ static bool process_pending()
 	int head = fs_list_get("data/outbox/HEAD");
 	int tail = fs_list_get("data/outbox/TAIL");
 	bool result = false;
-	for( int i = tail + 1; i <= head; ++i ) {
-		struct outbox_envelope* env = outbox_envelope_from_id( i );
-		if( !env ) {
-			// Envelope file doesn't exist, advance tail
-			fs_list_set( "data/outbox/TAIL", i );
-			tail += 1;
-		} else if( env ) {
-			if( env->sent && i == tail+1 ) {
-				// Envelope already sent, advance tail
+	bool do_fork = false;
+	//for( int i = tail + 1; i <= head; ++i ) {
+	for( int i = head; i > tail; --i ) {
+		if( !do_fork || !fork() ) {
+			struct outbox_envelope* env = outbox_envelope_from_id( i );
+			if( !env ) {
+				// Envelope file doesn't exist, advance tail
 				fs_list_set( "data/outbox/TAIL", i );
 				tail += 1;
-			} else if( process_envelope(env) ) {
-				printf( "Done with outbox/%d.json\n", i );
-				result = true;
+			} else if( env ) {
+				if( env->sent ) {
+					if( i == tail+1 ) {
+						// Envelope already sent, advance tail
+						fs_list_set( "data/outbox/TAIL", i );
+						tail += 1;
+					}
+				} else if( process_envelope(env) ) {
+					printf( "Done with outbox/%d.json\n", i );
+					result = true;
+				}
+				outbox_envelope_free(env);
+			}
+			fflush(stdout);
+			if( do_fork ) {
+				exit(EXIT_SUCCESS);
 			}
-			outbox_envelope_free(env);
+		} else {
+			usleep(50000);
 		}
-		fflush(stdout);
 	}
 
 	return result;

src/controller/pleroma_api.c

@@ -149,7 +149,7 @@ bool handle_read_notification( struct http_request* req )
 		if( 1 == sscanf( params.max_id, "%012d", &max_id ) ) {
 			for( int i = 0; i < 80; ++i ) {
 				struct notification* n = notification_from_id( max_id - i );
-				if( !n ) { break; }
+				if( !n ) { continue; }
 				if( n->is_seen ) { goto done_one_max_id; }
 
 				n->is_seen = true;
@@ -162,6 +162,8 @@ bool handle_read_notification( struct http_request* req )
 				done_one_max_id:
 				notification_free(n);
 			}
+		} else {
+			printf( "Failed to extract max_id from %s\n", params.max_id );
 		}
 		fprintf( f, "]" );
 	} else if( params.id ) {

src/http

@@ -1 +1 @@
-Subproject commit 8ff817b844739deb467de6525ab31c43a62e17ad
+Subproject commit 578aa0f81e7936be0138cf8682733f6e617d2cda

src/model/crypto/http_sign.c

@@ -24,14 +24,20 @@ bool http_signature_make( struct crypto_keys* keys, struct http_signature* sign
 
 	memset(sign,0,sizeof(*sign));
 
-	if( 0 != strncmp( "https://", inbox, 8 ) ) {
-		printf( "! Invalid inbox: %s\n", inbox );
-		return false;
-	}
+	char* path = NULL;
+	char* host = NULL;
 
 	// Separate host and path from inbox
-	char* path = index( &inbox[8], '/' );
-	char* host = sign->host = strndup( &inbox[8], path - &inbox[8] );
+	if( 0 == strncmp( "https://", inbox, 8 ) ) {
+		path = index( &inbox[8], '/' );
+		host = sign->host = strndup( &inbox[8], path - &inbox[8] );
+	} else if( 0 == strncmp( "http://", inbox, 7 ) ) {
+		path = index( &inbox[7], '/' );
+		host = sign->host = strndup( &inbox[7], path - &inbox[7] );
+	} else {
+		printf( "Unknown protocol for URL %s\n", inbox );
+		return false;
+	}
 
 	// Build HTTP date  - TODO: move to separate file
 	time_t utc_time = time(NULL);

src/model/fetch.c

@@ -33,7 +33,7 @@ size_t fetch_handle_header( char* header, size_t size, size_t nitems, void* user
 		return result;
 	}
 
-	printf( "? Header: |%.*s|\n", bytes, header );
+	//printf( "? Header: |%.*s|\n", bytes, header );
 
 	if( 0 != strncmp("onion-location: http://",header,sizeof("onion-location: http://")-1 ) ) {
 		return result;
@@ -58,7 +58,7 @@ size_t fetch_handle_header( char* header, size_t size, size_t nitems, void* user
 		pos -= 1;
 	}
 
-	printf( "+ Onion Host: |%s|\n", onion_host );
+	//printf( "+ Onion Host: |%s|\n", onion_host );
 	if( fd->p ) {
 		printf( "Updating peer onion host to %s\n", onion_host );
 		free(fd->p->tor_hidden_service);

src/model/peer.c

@@ -7,6 +7,7 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <ctype.h>
 #include <sys/stat.h>
 
 #define PEERS_BY_DOMAIN "data/peers/by_domain"
@@ -18,6 +19,8 @@ static struct json_object_field peer_layout[] = {
 
 	JSON_FIELD_DATETIME( last_successful_delivery, false ),
 	JSON_FIELD_DATETIME( last_failed_delivery, false ),
+	JSON_FIELD_DATETIME( last_hidden_service_delivery, false ),
+	JSON_FIELD_DATETIME( last_tor_delivery, false ),
 
 	JSON_FIELD_BOOL( admin_disable_outbox, false ),
 
@@ -59,6 +62,12 @@ struct peer* peer_from_domain( const char* domain )
 {
 	struct peer* res = NULL;
 
+	for( const char* p = domain; *p; ++p ) {
+		if( !isprint(*p) ) {
+			return NULL;
+		}
+	}
+
 	char* id_str = ffdb_trie_get( PEERS_BY_DOMAIN, domain );
 	if( !id_str ) { return NULL; }
 

src/model/peer.h

@@ -12,6 +12,9 @@ struct peer
 	time_t last_successful_delivery;
 	time_t last_failed_delivery;
 	bool admin_disable_outbox;
+
+	time_t last_hidden_service_delivery;
+	time_t last_tor_delivery;
 };
 
 void peer_model_init();

src/model/server.c

@@ -21,11 +21,10 @@ static struct json_object_field app_args_layout[] = {
 		.offset = offsetof( OBJ_TYPE, http_settings.bind_port ),
 		.type = &json_field_integer,
 	},
-	//JSON_FIELD_STRING( addr, false ),
-	//JSON_FIELD_INTEGER( port, false ),
 	JSON_FIELD_BOOL( debug, false ),
 	JSON_FIELD_BOOL( disable_tor, false ),
 	JSON_FIELD_INTEGER( tor_socks_port, false ),
+	JSON_FIELD_INTEGER( outbox_discard_limit, false ),
 
 	JSON_FIELD_BOOL( develop, false ),
 	JSON_FIELD_END,
@@ -61,6 +60,7 @@ struct app_args* app_args_new( int argc, char** argv )
 	args->http_settings.bind_address = strdup("0.0.0.0");
 	args->tor_socks_port = 9123;
 	args->section = -1;
+	args->outbox_discard_limit = 5;
 
 	json_read_object_layout_from_file( "data/server.json", app_args_layout, args );
 

src/model/server.h

@@ -17,6 +17,8 @@ struct app_args
 	bool disable_tor;
 	int tor_socks_port;
 
+	int outbox_discard_limit;
+
 	bool develop;
 };
 

src/model/status.c

@@ -545,6 +545,9 @@ struct status* status_fetch_from_uri( const char* uri )
 
 	return s;
 }
+
+static int system_status_id = 1;
+
 struct status* status_new_system_unfollow( int account_id )
 {
 	struct account* a = account_from_id(account_id);
@@ -554,7 +557,8 @@ struct status* status_new_system_unfollow( int account_id )
 	s = malloc(sizeof(*s));
 	memset(s,0,sizeof(*s));
 
-	s->id = -1;
+	system_status_id += 1;
+	s->id = -system_status_id;
 	s->published = time(NULL);
 	s->account_id = -1;
 	asprintf( &s->content, "%s unfollowed you\n", a->display_name );
@@ -572,7 +576,8 @@ struct status* status_new_system_block( int account_id )
 	s = malloc(sizeof(*s));
 	memset(s,0,sizeof(*s));
 
-	s->id = -1;
+	system_status_id += 1;
+	s->id = -system_status_id;
 	s->published = time(NULL);
 	s->account_id = -1;
 	asprintf( &s->content, "%s blocked you. View their account <a href='%s'>here</a>.", a->display_name, a->account_url );

src/rsa-signature-2017

@@ -1 +1 @@
-Subproject commit 9d574afe5ed85c64fc006ffbc2c78aafa6d2503f
+Subproject commit cc87290e39a9fc89c9a4e27bceafe3ab9c4f34b6