From 845666b3d5feb807d2686de0e146c033a0cfb73c Mon Sep 17 00:00:00 2001 From: teknomunk Date: Thu, 27 Jul 2023 18:10:40 -0500 Subject: [PATCH] Fix typo in README, update submodules, implement delivery of activities via TOR, make sure system statuses have unique IDs, fix HTTP signature generation to work with http:// URLs --- README.md | 2 +- src/controller/fetch.c | 4 +- src/controller/outbox.c | 260 ++++++++++++++++++++++++----------- src/controller/pleroma_api.c | 4 +- src/http | 2 +- src/model/crypto/http_sign.c | 18 ++- src/model/fetch.c | 4 +- src/model/peer.c | 9 ++ src/model/peer.h | 3 + src/model/server.c | 4 +- src/model/server.h | 2 + src/model/status.c | 9 +- src/rsa-signature-2017 | 2 +- 13 files changed, 222 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 1cab9df..9a47167 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# apoge +# apogee Personal ActivityPub server intended for single-user instances. diff --git a/src/controller/fetch.c b/src/controller/fetch.c index 5876bc6..a3568a3 100644 --- a/src/controller/fetch.c +++ b/src/controller/fetch.c @@ -29,11 +29,11 @@ static void process_fetch_once() char* key = NULL; char* value = NULL; if( !ffdb_trie_get_index( STUBS, idx, &key, &value ) ) { - printf( "Unable to get STUBS[%d], skipping\n", idx ); + //printf( "Unable to get STUBS[%d], skipping\n", idx ); continue; } - printf( "STUBS[%d]: %s = %s\n", idx, key, value ); + //printf( "STUBS[%d]: %s = %s\n", idx, key, value ); struct status* s = status_from_id( atoi(key) ); if( !s ) { printf( "Unable to load status with id %s\n", key ); diff --git a/src/controller/outbox.c b/src/controller/outbox.c index 5be3415..fcbcc32 100644 --- a/src/controller/outbox.c +++ b/src/controller/outbox.c @@ -19,6 +19,7 @@ // Submodules #include "http/url.h" +#include "util/format.h" // Stdlib #include @@ -67,6 +68,102 @@ cleanup: return result; } +static long submit_activity( const char* postdata, const char* inbox, const char* deliver, struct crypto_keys* keys, struct fetch_data* fd, bool use_tor_proxy ) +{ + printf( "Using inbox=%s\n", inbox ); + printf( "Using deliver=%s\n", deliver ); + printf( "Using proxy=%c\n", use_tor_proxy ? 'Y' : 'N' ); + + struct http_signature hs; + memset( &hs, 0, sizeof(hs) ); + hs.input.url = deliver; + hs.input.method = "post"; + hs.input.post_data = postdata; + if( !http_signature_make( keys, &hs ) ) { + printf( "! Failed to make HTTP signature\n" ); + return 600; + } + + char date_header[512]; + snprintf( date_header, sizeof(date_header), "Date: %s", hs.date ); + + char sign_header[512]; + snprintf( sign_header, sizeof(sign_header), "Signature: keyId=\"https://%s/owner/actor#mainKey\",headers=\"(request-target) host date content-length digest\",signature=\"%s\"", + g_server->domain, + hs.signature + ); + + char user_agent[512]; + snprintf( user_agent, sizeof(user_agent), "User-Agent: curl (Apogee/0.1; +https://%s/owner/actor)", g_server->domain ); + + char digest_header[512]; + snprintf( digest_header, sizeof(digest_header), "Digest: %s", hs.digest ); + + char content_length_header[512]; + snprintf( content_length_header, sizeof(content_length_header), "Content-Length: %d", hs.content_length ); + + char proxy[512]; + snprintf( proxy,512, "socks5h://localhost:%d", g_server->tor_socks_port ); + + printf( "Performing post to %s\n", deliver ); + long status_code = -1; + const void* request_clearnet[] = { + HTTP_REQ_URL, deliver, + HTTP_REQ_HEADER, user_agent, + HTTP_REQ_HEADER, date_header, + HTTP_REQ_HEADER, sign_header, + HTTP_REQ_HEADER, content_length_header, + HTTP_REQ_HEADER, digest_header, + HTTP_REQ_HEADER, "Content-Type: application/activity+json", + HTTP_REQ_POSTDATA, postdata, + HTTP_REQ_RESULT_STATUS, &status_code, + HTTP_RES_HEADER_CALLBACK, fetch_handle_header, fd, + //HTTP_REQ_PROXY, proxy, + NULL + }; + + const void* request_tor[] = { + HTTP_REQ_URL, deliver, + HTTP_REQ_HEADER, user_agent, + HTTP_REQ_HEADER, date_header, + HTTP_REQ_HEADER, sign_header, + HTTP_REQ_HEADER, content_length_header, + HTTP_REQ_HEADER, digest_header, + HTTP_REQ_HEADER, "Content-Type: application/activity+json", + HTTP_REQ_POSTDATA, postdata, + HTTP_REQ_RESULT_STATUS, &status_code, + HTTP_RES_HEADER_CALLBACK, fetch_handle_header, fd, + HTTP_REQ_PROXY, proxy, + HTTP_REQ_TIMEOUT, (void*)10, + NULL + }; + + // POST to inbox + const void** request = request_clearnet; + if( use_tor_proxy ) { + request = request_tor; + } + + printf( "\n\nRequest result:\n" ); + fflush(stdout); + http_client_do(request); + printf( "\n\n" ); + fflush(stdout); + + http_signature_free( &hs ); + + return status_code; +} +static bool delivery_succeeded( long status_code ) +{ + switch( status_code ) { + case 200: + case 202: + return true; + } + return false; +} + static bool process_envelope( struct outbox_envelope* env ) { bool result = false; @@ -76,6 +173,7 @@ static bool process_envelope( struct outbox_envelope* env ) struct ap_object* act = NULL; struct account* to_account = NULL; struct peer* p = NULL; + char* tor_inbox = NULL; if( env->sent ) { return false; } if( env->retry_after > time(NULL) ) { return false; } @@ -99,6 +197,15 @@ static bool process_envelope( struct outbox_envelope* env ) url_get_domain( inbox, domain,sizeof(domain) ); p = peer_create_from_domain( domain ); + // Handle tor delivery + //* + if( p->tor_hidden_service ) { + char* path = strchr( inbox + 9, '/' ); + tor_inbox = aformat( "http://%s%s", p->tor_hidden_service, path ); + printf( "Using TOR inbox: %s\n", tor_inbox ); + } + //*/ + struct fetch_data fd; fd.p = p; @@ -148,87 +255,62 @@ static bool process_envelope( struct outbox_envelope* env ) postdata[size] = '\0'; printf( "post: %s\n", postdata ); - struct http_signature hs; - memset( &hs, 0, sizeof(hs) ); - hs.input.url = inbox; - hs.input.method = "post"; - hs.input.post_data = postdata; - if( !http_signature_make( keys, &hs ) ) { - printf( "! Failed to make HTTP signature\n" ); - goto failed; - } - - char date_header[512]; - snprintf( date_header, sizeof(date_header), "Date: %s", hs.date ); - - char sign_header[512]; - snprintf( sign_header, sizeof(sign_header), "Signature: keyId=\"https://%s/owner/actor#mainKey\",headers=\"(request-target) host date content-length digest\",signature=\"%s\"", - g_server->domain, - hs.signature - ); + long status_code; - char user_agent[512]; - snprintf( user_agent, sizeof(user_agent), "User-Agent: curl (Apogee/0.1; +https://%s/owner/actor)", g_server->domain ); - - char digest_header[512]; - snprintf( digest_header, sizeof(digest_header), "Digest: %s", hs.digest ); - - char content_length_header[512]; - snprintf( content_length_header, sizeof(content_length_header), "Content-Length: %d", hs.content_length ); + // Try TOR Hidden Service delivery + if( tor_inbox ) { + status_code = submit_activity( postdata, inbox, tor_inbox, keys, &fd, true ); + if( delivery_succeeded(status_code) ) { + p->last_hidden_service_delivery = time(NULL); + goto success; + } + } - printf( "Performing post to %s\n", inbox ); - long status_code = -1; - const void* request[] = { - HTTP_REQ_URL, inbox, - HTTP_REQ_HEADER, user_agent, - HTTP_REQ_HEADER, date_header, - HTTP_REQ_HEADER, sign_header, - HTTP_REQ_HEADER, content_length_header, - HTTP_REQ_HEADER, digest_header, - HTTP_REQ_HEADER, "Content-Type: application/activity+json", - HTTP_REQ_POSTDATA, postdata, - HTTP_REQ_RESULT_STATUS, &status_code, - HTTP_RES_HEADER_CALLBACK, fetch_handle_header, &fd, - NULL - }; + // Try TOR delivery + status_code = submit_activity( postdata, inbox, inbox, keys, &fd, true ); + if( delivery_succeeded(status_code) ) { + p->last_tor_delivery = time(NULL); + goto success; + } - // POST to inbox - printf( "\n\nRequest result:\n" ); - fflush(stdout); - http_client_do(request); - printf( "\n\n" ); - fflush(stdout); + // Try clearnet delivery + status_code = submit_activity( postdata, inbox, inbox, keys, &fd, false ); - http_signature_free( &hs ); + // delivery failed + printf( "\nServer returned status code %ld\n", status_code ); + if( env->retry_after ) { + env->retries += 1; + } + if( p ) { + p->last_failed_delivery = time(NULL); - if( status_code == 200 || status_code == 202 ) { - printf( "Submitted successfully\n" ); - if( p ) { - p->last_successful_delivery = time(NULL); - peer_save(p); + if( p->last_successful_delivery - p->last_failed_delivery > 60*60*24*7 ) { + // At least one week since last successful delivery + printf( "Over a week since last successful delivery, marking for discard.\n" ); + env->retries = g_server->outbox_discard_limit; } + + peer_save(p); + } + if( env->retries >= g_server->outbox_discard_limit ) { + // Force discard after 10 delivery attempts + // TODO: change this to a configuration option goto discard; - } else { - printf( "\nServer returned status code %ld\n", status_code ); - if( env->retry_after ) { - env->retries += 1; - } - if( env->retries > 10 ) { - // Force discard after 10 delivery attempts - // TODO: change this to a configuration option - goto discard; - } - env->retry_after = time(NULL) + 60 * ( env->retries + 1 ) * ( env->retries + 1 ); - outbox_envelope_save(env); - if( p ) { - p->last_failed_delivery = time(NULL); - peer_save(p); - } } + env->retry_after = time(NULL) + 60 * ( env->retries + 1 ) * ( env->retries + 1 ); + outbox_envelope_save(env); goto failed; +success: + printf( "Submitted successfully\n" ); + if( p ) { + p->last_successful_delivery = time(NULL); + peer_save(p); + } + goto discard; cleanup: + free(tor_inbox); ap_object_free(act); account_free(to_account); free(postdata); @@ -242,6 +324,7 @@ failed: result = false; goto cleanup; discard: + printf( "Discarding.\n" ); env->sent = true; outbox_envelope_save(env); result = true; @@ -253,24 +336,35 @@ static bool process_pending() int head = fs_list_get("data/outbox/HEAD"); int tail = fs_list_get("data/outbox/TAIL"); bool result = false; - for( int i = tail + 1; i <= head; ++i ) { - struct outbox_envelope* env = outbox_envelope_from_id( i ); - if( !env ) { - // Envelope file doesn't exist, advance tail - fs_list_set( "data/outbox/TAIL", i ); - tail += 1; - } else if( env ) { - if( env->sent && i == tail+1 ) { - // Envelope already sent, advance tail + bool do_fork = false; + //for( int i = tail + 1; i <= head; ++i ) { + for( int i = head; i > tail; --i ) { + if( !do_fork || !fork() ) { + struct outbox_envelope* env = outbox_envelope_from_id( i ); + if( !env ) { + // Envelope file doesn't exist, advance tail fs_list_set( "data/outbox/TAIL", i ); tail += 1; - } else if( process_envelope(env) ) { - printf( "Done with outbox/%d.json\n", i ); - result = true; + } else if( env ) { + if( env->sent ) { + if( i == tail+1 ) { + // Envelope already sent, advance tail + fs_list_set( "data/outbox/TAIL", i ); + tail += 1; + } + } else if( process_envelope(env) ) { + printf( "Done with outbox/%d.json\n", i ); + result = true; + } + outbox_envelope_free(env); + } + fflush(stdout); + if( do_fork ) { + exit(EXIT_SUCCESS); } - outbox_envelope_free(env); + } else { + usleep(50000); } - fflush(stdout); } return result; diff --git a/src/controller/pleroma_api.c b/src/controller/pleroma_api.c index 4ca6560..6cf27cb 100644 --- a/src/controller/pleroma_api.c +++ b/src/controller/pleroma_api.c @@ -149,7 +149,7 @@ bool handle_read_notification( struct http_request* req ) if( 1 == sscanf( params.max_id, "%012d", &max_id ) ) { for( int i = 0; i < 80; ++i ) { struct notification* n = notification_from_id( max_id - i ); - if( !n ) { break; } + if( !n ) { continue; } if( n->is_seen ) { goto done_one_max_id; } n->is_seen = true; @@ -162,6 +162,8 @@ bool handle_read_notification( struct http_request* req ) done_one_max_id: notification_free(n); } + } else { + printf( "Failed to extract max_id from %s\n", params.max_id ); } fprintf( f, "]" ); } else if( params.id ) { diff --git a/src/http b/src/http index 8ff817b..578aa0f 160000 --- a/src/http +++ b/src/http @@ -1 +1 @@ -Subproject commit 8ff817b844739deb467de6525ab31c43a62e17ad +Subproject commit 578aa0f81e7936be0138cf8682733f6e617d2cda diff --git a/src/model/crypto/http_sign.c b/src/model/crypto/http_sign.c index 3986988..f3fd535 100644 --- a/src/model/crypto/http_sign.c +++ b/src/model/crypto/http_sign.c @@ -24,14 +24,20 @@ bool http_signature_make( struct crypto_keys* keys, struct http_signature* sign memset(sign,0,sizeof(*sign)); - if( 0 != strncmp( "https://", inbox, 8 ) ) { - printf( "! Invalid inbox: %s\n", inbox ); - return false; - } + char* path = NULL; + char* host = NULL; // Separate host and path from inbox - char* path = index( &inbox[8], '/' ); - char* host = sign->host = strndup( &inbox[8], path - &inbox[8] ); + if( 0 == strncmp( "https://", inbox, 8 ) ) { + path = index( &inbox[8], '/' ); + host = sign->host = strndup( &inbox[8], path - &inbox[8] ); + } else if( 0 == strncmp( "http://", inbox, 7 ) ) { + path = index( &inbox[7], '/' ); + host = sign->host = strndup( &inbox[7], path - &inbox[7] ); + } else { + printf( "Unknown protocol for URL %s\n", inbox ); + return false; + } // Build HTTP date - TODO: move to separate file time_t utc_time = time(NULL); diff --git a/src/model/fetch.c b/src/model/fetch.c index 219f185..9fa5c20 100644 --- a/src/model/fetch.c +++ b/src/model/fetch.c @@ -33,7 +33,7 @@ size_t fetch_handle_header( char* header, size_t size, size_t nitems, void* user return result; } - printf( "? Header: |%.*s|\n", bytes, header ); + //printf( "? Header: |%.*s|\n", bytes, header ); if( 0 != strncmp("onion-location: http://",header,sizeof("onion-location: http://")-1 ) ) { return result; @@ -58,7 +58,7 @@ size_t fetch_handle_header( char* header, size_t size, size_t nitems, void* user pos -= 1; } - printf( "+ Onion Host: |%s|\n", onion_host ); + //printf( "+ Onion Host: |%s|\n", onion_host ); if( fd->p ) { printf( "Updating peer onion host to %s\n", onion_host ); free(fd->p->tor_hidden_service); diff --git a/src/model/peer.c b/src/model/peer.c index 59585e8..2aa3eaa 100644 --- a/src/model/peer.c +++ b/src/model/peer.c @@ -7,6 +7,7 @@ #include #include +#include #include #define PEERS_BY_DOMAIN "data/peers/by_domain" @@ -18,6 +19,8 @@ static struct json_object_field peer_layout[] = { JSON_FIELD_DATETIME( last_successful_delivery, false ), JSON_FIELD_DATETIME( last_failed_delivery, false ), + JSON_FIELD_DATETIME( last_hidden_service_delivery, false ), + JSON_FIELD_DATETIME( last_tor_delivery, false ), JSON_FIELD_BOOL( admin_disable_outbox, false ), @@ -59,6 +62,12 @@ struct peer* peer_from_domain( const char* domain ) { struct peer* res = NULL; + for( const char* p = domain; *p; ++p ) { + if( !isprint(*p) ) { + return NULL; + } + } + char* id_str = ffdb_trie_get( PEERS_BY_DOMAIN, domain ); if( !id_str ) { return NULL; } diff --git a/src/model/peer.h b/src/model/peer.h index dfb548c..8d4cf8f 100644 --- a/src/model/peer.h +++ b/src/model/peer.h @@ -12,6 +12,9 @@ struct peer time_t last_successful_delivery; time_t last_failed_delivery; bool admin_disable_outbox; + + time_t last_hidden_service_delivery; + time_t last_tor_delivery; }; void peer_model_init(); diff --git a/src/model/server.c b/src/model/server.c index ec6bd13..646fe42 100644 --- a/src/model/server.c +++ b/src/model/server.c @@ -21,11 +21,10 @@ static struct json_object_field app_args_layout[] = { .offset = offsetof( OBJ_TYPE, http_settings.bind_port ), .type = &json_field_integer, }, - //JSON_FIELD_STRING( addr, false ), - //JSON_FIELD_INTEGER( port, false ), JSON_FIELD_BOOL( debug, false ), JSON_FIELD_BOOL( disable_tor, false ), JSON_FIELD_INTEGER( tor_socks_port, false ), + JSON_FIELD_INTEGER( outbox_discard_limit, false ), JSON_FIELD_BOOL( develop, false ), JSON_FIELD_END, @@ -61,6 +60,7 @@ struct app_args* app_args_new( int argc, char** argv ) args->http_settings.bind_address = strdup("0.0.0.0"); args->tor_socks_port = 9123; args->section = -1; + args->outbox_discard_limit = 5; json_read_object_layout_from_file( "data/server.json", app_args_layout, args ); diff --git a/src/model/server.h b/src/model/server.h index d751bc7..a4a1684 100644 --- a/src/model/server.h +++ b/src/model/server.h @@ -17,6 +17,8 @@ struct app_args bool disable_tor; int tor_socks_port; + int outbox_discard_limit; + bool develop; }; diff --git a/src/model/status.c b/src/model/status.c index da1f74a..98a171f 100644 --- a/src/model/status.c +++ b/src/model/status.c @@ -545,6 +545,9 @@ struct status* status_fetch_from_uri( const char* uri ) return s; } + +static int system_status_id = 1; + struct status* status_new_system_unfollow( int account_id ) { struct account* a = account_from_id(account_id); @@ -554,7 +557,8 @@ struct status* status_new_system_unfollow( int account_id ) s = malloc(sizeof(*s)); memset(s,0,sizeof(*s)); - s->id = -1; + system_status_id += 1; + s->id = -system_status_id; s->published = time(NULL); s->account_id = -1; asprintf( &s->content, "%s unfollowed you\n", a->display_name ); @@ -572,7 +576,8 @@ struct status* status_new_system_block( int account_id ) s = malloc(sizeof(*s)); memset(s,0,sizeof(*s)); - s->id = -1; + system_status_id += 1; + s->id = -system_status_id; s->published = time(NULL); s->account_id = -1; asprintf( &s->content, "%s blocked you. View their account here.", a->display_name, a->account_url ); diff --git a/src/rsa-signature-2017 b/src/rsa-signature-2017 index 9d574af..cc87290 160000 --- a/src/rsa-signature-2017 +++ b/src/rsa-signature-2017 @@ -1 +1 @@ -Subproject commit 9d574afe5ed85c64fc006ffbc2c78aafa6d2503f +Subproject commit cc87290e39a9fc89c9a4e27bceafe3ab9c4f34b6