From f545d51b50055668c11e3a6b82d0dfc460561c0c Mon Sep 17 00:00:00 2001 From: teknomunk Date: Fri, 18 Aug 2023 05:59:19 -0500 Subject: [PATCH] commit several updates --- src/ap | 2 +- src/controller/cli.c | 49 +++++- src/controller/fetch.c | 2 +- src/controller/inbox.c | 25 ++- src/controller/outbox.c | 31 ++-- src/controller/owner.c | 24 +++ src/controller/test/crypto.c | 4 +- src/ffdb | 2 +- src/json | 2 +- src/model/account.c | 78 ++++++++- src/model/account.h | 3 +- src/model/account/ap_data.c | 19 ++- src/model/account/ap_sync.c | 65 ++++--- src/model/crypto/http_sign.c | 42 +++-- src/model/crypto/http_sign.h | 22 ++- src/model/fetch.c | 247 ++++++++++++++++++--------- src/model/gc.c | 2 + src/model/peer.c | 1 + src/model/peer.h | 1 + src/model/status.c | 57 ++++++- src/model/status.h | 2 + src/view/litepub-0.1.jsonld.template | 71 ++++++++ 22 files changed, 600 insertions(+), 151 deletions(-) create mode 100644 src/view/litepub-0.1.jsonld.template diff --git a/src/ap b/src/ap index e239bec..70380d7 160000 --- a/src/ap +++ b/src/ap @@ -1 +1 @@ -Subproject commit e239bec5047699782d269fe6c33ed1aa87206734 +Subproject commit 70380d76001d6f7c82206fec5e0575fd3eb8b0d7 diff --git a/src/controller/cli.c b/src/controller/cli.c index 2e20124..c1db04d 100644 --- a/src/controller/cli.c +++ b/src/controller/cli.c @@ -18,6 +18,9 @@ // Controller #include "controller/inbox.h" +// Subcomponents +#include "ffdb/fs_list.h" + // Standard Library #include #include @@ -136,6 +139,25 @@ static bool handle_command_status_sync( struct cli_request* req ) status_free(s); return true; } +static bool handle_command_status_index( struct cli_request* req ) +{ + int res = cli_route_command( req, "index", 0, "" ); + if( res != 1 ) { return !!res; } + + int head = fs_list_get( "data/statuses/HEAD" ); + + for( int id = 1; id <= head; ++id ) { + struct status* s = status_from_id(id); + if( s ) { + printf( "Updating indexes for status %d/%d (%0.2g)\n", id, head, (double)id / head ); + // status_index(s); + status_save(s); + status_free(s); + } + } + + return true; +} static bool handle_command_status( struct cli_request* req ) { int res = cli_route_command( req, "status", 1, "(import|sync) ..." ); @@ -143,6 +165,7 @@ static bool handle_command_status( struct cli_request* req ) if( handle_command_status_import( req ) ) { return true; } if( handle_command_status_sync( req ) ) { return true; } + if( handle_command_status_index( req ) ) { return true; } printf( "Usage: %s status (import|sync)", req->binary_name ); return true; @@ -187,8 +210,15 @@ bool handle_command_account_sync( struct cli_request* req, int account_id ) int res = cli_route_command( req, "sync", 0, "" ); if( res != 1 ) { return !!res; } + bool force = false; + if( req->argc >= 1 ) { + if( 0 == strcmp("--force",req->argv[0]) ) { + force = true; + } + } + printf( "Syncing account %d\n", account_id ); - account_sync_from_activity_pub( account_id ); + account_sync_from_activity_pub( account_id, force ); return true; } @@ -229,6 +259,17 @@ bool handle_command_pull( struct cli_request* req ) account_free(owner); return true; } +bool handle_command_account_import( struct cli_request* req ) +{ + if( req->argc < 1 ) { return false; } + + struct account* a = account_from_uri( req->argv[0] ); + if( a ) { + printf( "Account %s imported as id %d\n", req->argv[0], a->id ); + } + account_free(a); + return a != NULL; +} bool handle_command_account( struct cli_request* req ) { @@ -237,6 +278,12 @@ bool handle_command_account( struct cli_request* req ) bool result = false; + if( 0 == strcmp("import",req->argv[0]) ) { + req->argv += 1; + req->argc -= 1; + return handle_command_account_import(req); + } + int account_id = atoi(req->argv[0]); req->argv += 1; req->argc -= 1; diff --git a/src/controller/fetch.c b/src/controller/fetch.c index a3568a3..5006469 100644 --- a/src/controller/fetch.c +++ b/src/controller/fetch.c @@ -68,7 +68,7 @@ static void process_account_sync_once() if( !a ) { continue; } if( time(NULL) >= a->next_update ) { - account_sync_from_activity_pub( a->id ); + account_sync_from_activity_pub( a->id, true ); } else { char buffer[512]; rfc3339_time_string( a->next_update, buffer,512 ); diff --git a/src/controller/inbox.c b/src/controller/inbox.c index ec26927..871475e 100644 --- a/src/controller/inbox.c +++ b/src/controller/inbox.c @@ -416,6 +416,7 @@ static bool route_create( struct ap_object* act ) // Create local status s = status_from_uri( obj->id ); if( !s ) { + printf( "Creating status from activity...\n" ); s = status_from_activity(obj); if( !s ) { printf( "! Failed to load status from activity\n" ); @@ -575,8 +576,10 @@ static bool process_one() } fflush(stdout); - printf( "Handling forwarding\n" ); - if( handle_forward( env, act ) ) { goto discard; } + if( handle_forward( env, act ) ) { + printf( "Handling forwarding...discarding.\n" ); + goto discard; + } // Sanitize actor if( !act->actor ) { @@ -589,13 +592,25 @@ static bool process_one() // Discard delete requests if( act->type == ap_Delete ) { + printf( "Discarding delete.\n" ); goto discard; } // Validate signature - env->validated = http_signature_validate( env, "post /inbox", act->actor ); - - if( !env->validated ) { goto discard; } + struct hsv_result result2 = http_signature_validate( env, "post /inbox", act->actor ); + env->validated = result2.pass; + + if( !env->validated ) { + switch( result2.error ) { + case HSV_INVALID_SIGNATURE: + printf( "HTTP Signature not validated, discarding.\n" ); + goto discard; + case HSV_ACTOR_MISMATCH: + goto discard; + default: + goto failed; + } + } printf( "Processing %d\n", id ); if( !route_activity( act ) ) { diff --git a/src/controller/outbox.c b/src/controller/outbox.c index 698e870..b73c456 100644 --- a/src/controller/outbox.c +++ b/src/controller/outbox.c @@ -118,7 +118,6 @@ static long submit_activity( const char* postdata, const char* inbox, const char HTTP_REQ_POSTDATA, postdata, HTTP_REQ_RESULT_STATUS, &status_code, HTTP_RES_HEADER_CALLBACK, fetch_handle_header, fd, - //HTTP_REQ_PROXY, proxy, NULL }; @@ -257,22 +256,24 @@ static bool process_envelope( struct outbox_envelope* env ) long status_code; - // Try TOR Hidden Service delivery - if( tor_inbox ) { - status_code = submit_activity( postdata, inbox, tor_inbox, keys, &fd, true ); + if( !p->admin_disable_tor ) { + // Try TOR Hidden Service delivery + if( tor_inbox ) { + status_code = submit_activity( postdata, inbox, tor_inbox, keys, &fd, true ); + if( delivery_succeeded(status_code) ) { + p->last_hidden_service_delivery = time(NULL); + goto success; + } + } + + // Try TOR delivery + status_code = submit_activity( postdata, inbox, inbox, keys, &fd, true ); if( delivery_succeeded(status_code) ) { - p->last_hidden_service_delivery = time(NULL); + p->last_tor_delivery = time(NULL); goto success; } } - // Try TOR delivery - status_code = submit_activity( postdata, inbox, inbox, keys, &fd, true ); - if( delivery_succeeded(status_code) ) { - p->last_tor_delivery = time(NULL); - goto success; - } - // Try clearnet delivery status_code = submit_activity( postdata, inbox, inbox, keys, &fd, false ); @@ -337,8 +338,7 @@ static bool process_pending() int tail = fs_list_get("data/outbox/TAIL"); bool result = false; - int limit = 10; - for( int i = head; (i > tail ) && ( limit > 0 ); --i ) { + for( int i = head; i > tail; --i ) { struct outbox_envelope* env = outbox_envelope_from_id( i ); if( !env ) { if( i == tail+1 ) { @@ -351,10 +351,9 @@ static bool process_pending() if( env->sent ) { outbox_envelope_delete(env); } else if( process_envelope(env) ) { - limit -= 1; printf( "Done with outbox/%d.json\n", i ); - result = true; outbox_envelope_delete(env); + return true; } } fflush(stdout); diff --git a/src/controller/owner.c b/src/controller/owner.c index 5493d65..d4f5264 100644 --- a/src/controller/owner.c +++ b/src/controller/owner.c @@ -3,6 +3,7 @@ // Submodules #include "http/server/request.h" #include "ap/object.h" +#include "ap/http.h" // Model #include "model/server.h" @@ -118,6 +119,27 @@ static bool handle_owner_actor( struct http_request* req ) account_free( owner_account ); return true; } +static bool handle_owner_account_redirect( struct http_request* req ) +{ + bool result = false; + struct account* owner_account = account_from_id(0); + if( !http_request_route( req, owner_account->handle ) ) { goto no_match; } + if( !http_request_route( req, "@" ) ) { goto no_match; } + if( !http_request_route( req, owner_account->server ) ) { goto no_match; } + + // Match + if( ap_object_handle_http_request_should_provide( req ) ) { + // TODO: redirect + } + result = false; + +cleanup: + account_free( owner_account ); + return result; +no_match: + result = false; + goto cleanup; +} static bool handle_avatar( struct http_request* req ) { @@ -146,6 +168,8 @@ bool route_owner( struct http_request* req ) if( http_request_route_term( req, "" ) ) { return handle_owner_actor(req); } + } else if( http_request_route( req, "/@" ) ) { + return handle_owner_account_redirect(req); } else if( http_request_route( req, "/following" ) ) { return handle_following(req); } else if( http_request_route( req, "/followers" ) ) { diff --git a/src/controller/test/crypto.c b/src/controller/test/crypto.c index e5d3f3e..3b25309 100644 --- a/src/controller/test/crypto.c +++ b/src/controller/test/crypto.c @@ -70,7 +70,7 @@ static bool test_http_signature() .body = "{\"hello\": \"world\"}", }; - return http_signature_validate( &env, "post /foo?param=value&pet=dog", "Test" ); + return http_signature_validate( &env, "post /foo?param=value&pet=dog", "Test" ).pass; } static bool test_http_signature_2() { @@ -113,7 +113,7 @@ static bool test_http_signature_2() .body = "{\"hello\": \"world\"}", }; - result = http_signature_validate( &env, "post /inbox", "Test" ); + result = http_signature_validate( &env, "post /inbox", "Test" ).pass; cleanup: http_signature_free(&hs); crypto_keys_free(keys); diff --git a/src/ffdb b/src/ffdb index 06066e9..87d5d24 160000 --- a/src/ffdb +++ b/src/ffdb @@ -1 +1 @@ -Subproject commit 06066e9de27b265e1191de272a67e5e4ba2353f8 +Subproject commit 87d5d2462561348fd8efaa096aba620e93552a8d diff --git a/src/json b/src/json index 31be475..d4b6e91 160000 --- a/src/json +++ b/src/json @@ -1 +1 @@ -Subproject commit 31be475dc44c76e889c31d1a74b6ada25984951c +Subproject commit d4b6e913836a7803d942323236d160db58ecc6c0 diff --git a/src/model/account.c b/src/model/account.c index 5d19822..dfc3d9c 100644 --- a/src/model/account.c +++ b/src/model/account.c @@ -32,6 +32,7 @@ #include #include #include +#include static struct json_enum account_types_enum[] = { { "owner", at_owner }, @@ -120,6 +121,7 @@ static struct json_object_field account_layout[] = { }, JSON_FIELD_INTEGER( follow_activity, false ), JSON_FIELD_STRING( account_url, true ), + JSON_FIELD_STRING( account_id, false ), JSON_FIELD_END, }; @@ -179,15 +181,23 @@ static struct account* account_load_from_id( int id, int recurse_limit ) } char filename[512]; - snprintf( filename, 512, "data/accounts/%d.json", id ); struct account* a = account_new(); a->id = id; + snprintf( filename, 512, "data/accounts/%d/data.json", id ); if( !json_read_object_layout_from_file( filename, account_layout, a ) ) { account_free(a); - return NULL; + a = account_new(); + a->id = id; + + snprintf( filename, 512, "data/accounts/%d.json", id ); + if( !json_read_object_layout_from_file( filename, account_layout, a ) ) { + account_free(a); + return NULL; + } } + if( a->replaced_by && a->replaced_by != a->id ) { int new_id = a->replaced_by; account_free(a); @@ -215,10 +225,54 @@ static bool index_uri_to_account_id( const char* uri, int account_id ) char id_str[32]; snprintf( id_str,32, "%d", account_id ); + char* old_id = ffdb_trie_get( "data/accounts/by-uri", uri ); + if( old_id ) { + if( atoi(old_id) < account_id ) { + printf( "Existing account at id=%s\n", old_id ); + free( old_id ); + return true; + } + free( old_id ); + } + ffdb_trie_set( "data/accounts/by-uri", uri, id_str ); - return hash_index_set( "data/accounts/uri_index/", uri, account_id ); + hash_index_remove( "data/accounts/uri_index/", uri ); + //return hash_index_set( "data/accounts/uri_index/", uri, account_id ); + return true; } +/* +static int account_search_for_id_from_uri( const char* uri ) +{ + printf( "Searching for account id for %s\n", uri ); + int max_account_id = fs_list_get( "data/accounts/HEAD" ); + int result = -1; + for( int i = 0; (result == -1) && (i < max_account_id); ++i ) { + struct account* a = account_from_id(i); + if( !a ) { continue; } + + if( !a->account_id ) { + // Force syncing + account_sync_from_activity_pub( i, true ); + } + + if( a->account_url && (0 == strcmp(a->account_url,uri)) ) { + result = a->id; + account_save(a); + } else if( a->account_id && (0 == strcmp(a->account_id,uri)) ) { + result = a->id; + account_save(a); + } + + account_free(a); + } + + printf( "result is %d\n", result ); + + return result; +} +//*/ + int account_id_from_uri( const char* uri ) { char* id_str = ffdb_trie_get( "data/accounts/by-uri", uri ); @@ -230,6 +284,7 @@ int account_id_from_uri( const char* uri ) int result = 0; if( !hash_index_get( "data/accounts/uri_index/", uri, &result ) ) { + //return account_search_for_id_from_uri( uri ); return -1; } @@ -275,6 +330,9 @@ struct account* account_from_uri( const char* uri ) account_free(a); return NULL; } + + index_uri_to_account_id( a->account_url, a->id ); + return a; } struct account* account_from_uri_or_fetch( const char* uri ) @@ -398,7 +456,7 @@ struct account* account_fetch_from_uri( const char* uri ) account_free(a); // Fail if we can't sync - if( !account_sync_from_activity_pub( account_id ) ) { + if( !account_sync_from_activity_pub( account_id, false ) ) { printf( "Failed to sync from activity pub data (account_id=%d)\n", account_id ); return NULL; } @@ -415,6 +473,7 @@ void account_free( struct account* a ) free(a->display_name); free(a->account_url); + free(a->account_id); free(a->inbox); free(a->shared_inbox); @@ -448,16 +507,25 @@ void account_free( struct account* a ) void account_save( struct account* a ) { char filename[512]; - snprintf( filename, 512, "data/accounts/%d.json", a->id ); + snprintf( filename, 512, "data/accounts/%d/data.json", a->id ); + printf( "Saving to filename %s\n", filename ); json_write_object_layout_to_file( filename, "\t", account_layout, a ); + snprintf( filename, 512, "data/accounts/%d.json", a->id ); + unlink( filename ); + // Index by uri if( a->account_url ) { char id_str[32]; snprintf( id_str,32, "%d", a->id ); ffdb_trie_set( "data/accounts/by-uri", a->account_url, id_str ); hash_index_remove( "data/accounts/uri_index/", a->account_url ); + + if( a->account_id ) { + ffdb_trie_set( "data/accounts/by-uri", a->account_id, id_str ); + hash_index_remove( "data/accounts/uri_index/", a->account_id ); + } } // Index by next_update diff --git a/src/model/account.h b/src/model/account.h index 26ea8cf..26821a7 100644 --- a/src/model/account.h +++ b/src/model/account.h @@ -49,6 +49,7 @@ struct account int account_type; char* account_url; + char* account_id; char* inbox; char* shared_inbox; @@ -106,7 +107,7 @@ void account_free( struct account* a ); void account_save( struct account* a ); // Update from external activity pub data -bool account_sync_from_activity_pub( unsigned int id ); +bool account_sync_from_activity_pub( unsigned int id, bool force ); bool account_sync_from_activity( struct account* a, struct ap_object* act ); struct ap_object* account_get_activity_pub_data( struct account* a ); void account_sync_following_list( struct account* a ); diff --git a/src/model/account/ap_data.c b/src/model/account/ap_data.c index 64f825a..065877f 100644 --- a/src/model/account/ap_data.c +++ b/src/model/account/ap_data.c @@ -31,7 +31,7 @@ struct ap_object* account_activity_pub( struct account* a ) struct ap_object* account_ap_actor( struct account* a ) { struct ap_object* act = activity_new_local_activity(); - act->published = time(NULL); + act->ap_context.language = clang_undefined; act->type = ap_Person; act->id = aformat( "https://%s/owner/actor", g_server->domain ); act->name = strdup(a->display_name); @@ -43,14 +43,23 @@ struct ap_object* account_ap_actor( struct account* a ) act->featured = aformat("https://%s/owner/collections/featured", g_server->domain ); act->followers = aformat("https://%s/owner/followers", g_server->domain); act->following = aformat("https://%s/owner/following", g_server->domain); + act->also_known_as.items = malloc(1); + act->discoverable.visible = true; + act->discoverable.value = false; + act->manually_approves_followers.visible = true; + act->manually_approves_followers.value = false; + + act->tags.items = malloc(1); struct ap_object* avatar = ap_object_new(); avatar->type = ap_Image; ap_object_array_append_ref( &avatar->url, aformat( "https://%s/owner/avatar.blob", g_server->domain ) ); + avatar->media_type = strdup( "image/png" ); act->icon = avatar; struct ap_object* banner = ap_object_new(); banner->type = ap_Image; + banner->media_type = strdup( "image/png" ); ap_object_array_append_ref( &banner->url, aformat( "https://%s/owner/banner.blob", g_server->domain ) ); act->image = banner; @@ -67,9 +76,10 @@ struct ap_object* account_ap_actor( struct account* a ) fseek( f, 0, SEEK_END ); int size = ftell(f); fseek( f, 0, SEEK_SET ); - char* pem = malloc( size + 1 ); + char* pem = malloc( size + 2 ); fread( pem, 1, size, f ); - pem[size] = 0; + pem[size+0] = '\n'; + pem[size+1] = '\0'; key->public_key = pem; fclose(f); @@ -296,7 +306,8 @@ struct ap_object* account_ap_featured( struct account* a ) { struct ap_object* o = activity_new_local_activity(); o->type = ap_OrderedCollection; - o->published = time(NULL); + o->ap_context.language = clang_undefined; + //o->published = time(NULL); char buffer[512]; o->id = aformat( "https://%s/owner/collections/featured", g_server->domain ); o->total_items = ffdb_trie_count( format( buffer,512, "data/accounts/%d/timeline/pinned", a->id ) ); diff --git a/src/model/account/ap_sync.c b/src/model/account/ap_sync.c index ebd53db..1950ecf 100644 --- a/src/model/account/ap_sync.c +++ b/src/model/account/ap_sync.c @@ -18,49 +18,55 @@ struct ap_object* account_get_activity_pub_data( struct account* a ) { if( !a ) { return NULL; } - if( !a->account_url ) { - printf( "No account url\n" ); - goto skip_pull; - } char filename[512]; snprintf( filename, 512, "data/accounts/%d/ap.json", a->id ); - printf( "ap_object_from_file( %s )\n", filename ); - pull_remote_file_if_older( filename, a->account_url, 60*60*24*3 ); -skip_pull: + + if( a->account_id ) { + pull_remote_file_if_older( filename, a->account_id, 60*60*24*3 ); + } else if( a->account_url ) { + pull_remote_file_if_older( filename, a->account_url, 60*60*24*3 ); + } else { + printf( "? No URL to sync ap data from\n" ); + } + return ap_object_from_file( filename ); } -bool account_sync_from_activity_pub( unsigned int account_id ) +bool account_sync_from_activity_pub( unsigned int account_id, bool force ) { bool result = false; struct account* a = account_from_id(account_id); struct ap_object* obj = NULL; + printf( "account_sync_from_activity_pub( account_id=%d )\n", account_id ); + if( !a ) { - a = account_new(); - a->id = account_id; - } else if( a->next_update > time(NULL) ) { + printf( "! Failed to load account %d\n", account_id ); + goto failed; + } else if( !force && ( a->next_update > time(NULL) ) ) { + printf( "? No update required\n" ); goto succeeded; } else if( a->local ) { + printf( "? Is local account\n" ); goto succeeded; } obj = account_get_activity_pub_data(a); if( !obj ) { - printf( "Failed to load AP data for %s (%d)\n", a->account_url, a->id ); + printf( "! Failed to load AP data for %s (%d)\n", a->account_url, a->id ); goto failed; } - if( !account_sync_from_activity( a, obj ) ) { goto failed; } + if( !account_sync_from_activity( a, obj ) ) { + printf( "! Failed to sync from activity pub data\n" ); + goto failed; + } // If we got here, this account can't be a stub any more a->stub = false; a->next_update = time(NULL) + (60*60*24*3) + (rand() % (60*60*24*2)); // Next update in 3-5 days - account_save(a); - account_index_webfinger(a); - goto succeeded; succeeded: result = true; @@ -68,7 +74,9 @@ succeeded: cleanup: if( a ) { a->next_update = time(NULL) + (60*60*24*3) + (rand() % (60*60*24*2)); // Next update in 3-5 days + printf( "Account saved.\n" ); account_save(a); + account_index_webfinger(a); } ap_object_free(obj); account_free(a); @@ -81,15 +89,18 @@ failed: bool account_sync_from_activity( struct account* a, struct ap_object* obj ) { + printf( "account_sync_from_activity( a->id = %d, obj=%p )\n", a->id, obj ); if( obj->url.count > 0 ) { a->account_url = strdup(obj->url.items[0].ref); // TODO: make this more robust } else if( obj->id ) { a->account_url = strdup(obj->id); } else { - printf( "No id or url in AP data\n" ); + printf( "! No id or url in AP data\n" ); return false; } + a->account_id = strdup(obj->id); + for( int i = 0; i < a->aliases.count; ++i ) { free( a->aliases.items[i] ); } @@ -131,6 +142,12 @@ bool account_sync_from_activity( struct account* a, struct ap_object* obj ) char* discard; strtok_r(server_name,"/",&discard); + a->server = server_name; + } else if( 0 == strncmp( obj->id, "http://", 7 ) ) { + char* server_name = strdup(&obj->id[7]); + char* discard; + strtok_r(server_name,"/",&discard); + a->server = server_name; } @@ -169,6 +186,8 @@ bool account_sync_from_activity( struct account* a, struct ap_object* obj ) fclose(key_pem); } free(id); + } else { + printf( "? No public key for account\n" ); } return true; @@ -235,6 +254,8 @@ cleanup: void account_sync_followers_list( struct account* a ) { + struct ap_object* obj = NULL; + struct ap_object* followers = NULL; ap_object_set_fetch_callback( fetch_ap_object_with_delay ); struct { @@ -243,9 +264,14 @@ void account_sync_followers_list( struct account* a ) } existing; account_list_followers( a, 0, INT_MAX, &existing ); - struct ap_object* obj = account_get_activity_pub_data(a); + obj = account_get_activity_pub_data(a); + if( !obj ) { + printf( "Unable to get activity pub object for account id=%d\n", a->id ); + goto cleanup; + } + if( !obj->followers ) { goto cleanup; } - struct ap_object* followers = ap_collection_from_uri( obj->followers ); + followers = ap_collection_from_uri( obj->followers ); if( !followers ) { goto cleanup; } for( struct ap_object* page = followers->first.ptr; page; ap_collection_iterate(&page) ) { @@ -278,6 +304,7 @@ void account_sync_followers_list( struct account* a ) cleanup: ap_object_free(followers); + ap_object_free(obj); } void account_pull_friends_of_friends( struct account* a ) { diff --git a/src/model/crypto/http_sign.c b/src/model/crypto/http_sign.c index f3fd535..618259e 100644 --- a/src/model/crypto/http_sign.c +++ b/src/model/crypto/http_sign.c @@ -156,12 +156,15 @@ failed: goto cleanup; } -bool http_signature_validate( struct ap_envelope* env, const char* request_target, const char* expected_actor ) +struct hsv_result http_signature_validate( struct ap_envelope* env, const char* request_target, const char* expected_actor ) { char* signature_header = NULL; char* date_header = NULL; struct crypto_keys* keys = NULL; - bool result = false; + struct hsv_result result = { + .pass = false, + .error = HSV_UNKNOWN, + }; char* hash_line = NULL; struct account* actor = NULL; @@ -177,7 +180,14 @@ bool http_signature_validate( struct ap_envelope* env, const char* request_targe date_header = strdup(value); } } - if( !signature_header ) { return false; } + if( !signature_header ) { + struct hsv_result result = { + .pass = false, + .error = HSV_NO_HEADER, + }; + return result; + //return false; + } //printf( "Found Signature: %s\n", signature_header ); // Validate time @@ -221,15 +231,16 @@ bool http_signature_validate( struct ap_envelope* env, const char* request_targe } // Make sure we got all the required information - if( !signature ) { printf( "! No signature\n" ); goto failed; } - if( !headers ) { printf( "! No headers\n" ); goto failed; } - if( !key_id ) { printf( "! no key_id\n" ); goto failed; } + if( !signature ) { printf( "! No signature\n" ); result.error = HSV_NO_SIGNATURE; goto failed; } + if( !headers ) { printf( "! No headers\n" ); result.error = HSV_NO_HEADERS; goto failed; } + if( !key_id ) { printf( "! no key_id\n" ); result.error = HSV_NO_KEY_ID; goto failed; } // Get actor URI if( 0 == strcmp(key_id,"Test") ) { keys = crypto_keys_new(); if( !crypto_keys_load_public( keys, "assets/test.public.pem" ) ) { printf( "! Failed to load public key\n" ); + result.error = HSV_NO_PUBKEY; goto failed; } } else { @@ -237,11 +248,13 @@ bool http_signature_validate( struct ap_envelope* env, const char* request_targe char* actor_uri = strtok_r( key_id, "#", &key_name ); if( !actor_uri ) { printf( "! Failed to get actor\n" ); + result.error = HSV_MISSING_ACTOR; goto failed; } if( 0 != strcmp( actor_uri, expected_actor ) ) { printf( "! Signature doesn't match actor (Expected %s, got %s)\n", expected_actor, actor_uri ); + result.error = HSV_ACTOR_MISMATCH; goto failed; } @@ -249,12 +262,17 @@ bool http_signature_validate( struct ap_envelope* env, const char* request_targe actor = account_from_uri_or_fetch(actor_uri); if( !actor ) { printf( "! failed to load account for %s\n", actor_uri ); + result.error = HSV_NO_ACCOUNT; goto failed; } // Get the public key keys = account_get_public_key( actor, key_name ); - if( !keys ) { goto failed; } + if( !keys ) { + printf( "! Failed to find public key for actor=%s (id=%d), key_name=%s\n", actor->account_url, actor->id, key_name ); + result.error = HSV_NO_PUBKEY; + goto failed; + } } // Create the hash line @@ -272,6 +290,7 @@ bool http_signature_validate( struct ap_envelope* env, const char* request_targe } else if( 0 == strcmp(part,"digest") ) { if( !validate_body_digest( env ) ) { printf( "! Digest validation failed" ); + result.error = HSV_DIGEST_FAILED; goto failed; } print_header_line( hl, env, part ); @@ -287,13 +306,14 @@ bool http_signature_validate( struct ap_envelope* env, const char* request_targe //printf( "\nhash_line:\n%s|\n\n", hash_line ); //result = crypto_keys_verify( keys, raw_hash, 32, signature ); - result = crypto_keys_verify( keys, hash_line, hash_line_size, signature ); - if( !result ) { + if( !crypto_keys_verify( keys, hash_line, hash_line_size, signature ) ) { printf( "! Signature is not valid\n" ); + result.error = HSV_INVALID_SIGNATURE; goto failed; } - result = true; + result.pass = true; + result.error = HSV_NO_ERROR; cleanup: crypto_keys_free(keys); account_free(actor); @@ -302,7 +322,7 @@ cleanup: free(hash_line); return result; failed: - result = false; + result.pass = false; goto cleanup; } diff --git a/src/model/crypto/http_sign.h b/src/model/crypto/http_sign.h index addf4a3..8cd8bdb 100644 --- a/src/model/crypto/http_sign.h +++ b/src/model/crypto/http_sign.h @@ -21,7 +21,27 @@ struct http_signature int content_length; }; +struct hsv_result +{ + bool pass; + int error; +}; +enum { + HSV_NO_ERROR = 0, + HSV_NO_HEADER = 1, + HSV_NO_SIGNATURE = 2, + HSV_NO_HEADERS = 3, + HSV_NO_KEY_ID = 4, + HSV_NO_PUBKEY = 5, + HSV_MISSING_ACTOR = 6, + HSV_ACTOR_MISMATCH = 7, + HSV_NO_ACCOUNT = 8, + HSV_DIGEST_FAILED = 9, + HSV_INVALID_SIGNATURE = 10, + HSV_UNKNOWN = 99999, +}; + bool http_signature_make( struct crypto_keys* keys, struct http_signature* sign ); void http_signature_free( struct http_signature* sign ); -bool http_signature_validate( struct ap_envelope* env, const char* request_target, const char* expected_actor ); +struct hsv_result http_signature_validate( struct ap_envelope* env, const char* request_target, const char* expected_actor ); diff --git a/src/model/fetch.c b/src/model/fetch.c index 9fa5c20..c38dfdd 100644 --- a/src/model/fetch.c +++ b/src/model/fetch.c @@ -70,21 +70,16 @@ size_t fetch_handle_header( char* header, size_t size, size_t nitems, void* user return result; } - -static bool do_fetch( const char* uri, FILE* result ) +static bool do_fetch_tor( const char* uri, struct fetch_data* fd, const char* result_filename ) { char user_agent[512]; snprintf( user_agent, sizeof(user_agent), "User-Agent: curl (Apogee/0.1; +https://%s/owner/actor)", g_server->domain ); - // Setup fetch data - struct fetch_data fd; - memset(&fd,0,sizeof(fd)); - { - char host_domain[512]; - if( url_get_domain( uri, host_domain, sizeof(host_domain) ) ) { - fd.p = peer_from_domain(host_domain); - } - } + char proxy[512]; + snprintf( proxy,512, "socks5h://localhost:%d", g_server->tor_socks_port ); + + FILE* result = fopen(result_filename,"w"); + if( !result ) { return false; } long status_code; const void* request[] = { @@ -93,97 +88,197 @@ static bool do_fetch( const char* uri, FILE* result ) HTTP_REQ_HEADER, user_agent, HTTP_REQ_OUTFILE, result, HTTP_REQ_RESULT_STATUS, &status_code, - HTTP_RES_HEADER_CALLBACK, fetch_handle_header, (void*)&fd, + HTTP_RES_HEADER_CALLBACK, fetch_handle_header, (void*)fd, + HTTP_REQ_PROXY, proxy, + HTTP_REQ_TIMEOUT, (void*)10, NULL, }; printf( "GET %s\n", uri ); http_client_do( request ); printf( "GET %s -> %ld\n", uri, status_code ); - // Save peer data - if( fd.p ) { - peer_save( fd.p ); - peer_free( fd.p ); + fclose(result); + + if( status_code != 200 ) { + unlink(result_filename); } + return status_code == 200; +} +static bool do_fetch_clearnet( const char* uri, struct fetch_data* fd, const char* result_filename ) +{ + char user_agent[512]; + snprintf( user_agent, sizeof(user_agent), "User-Agent: curl (Apogee/0.1; +https://%s/owner/actor)", g_server->domain ); + + FILE* result = fopen( result_filename, "w" ); + if( !result ) { return false; } + + long status_code; + const void* request[] = { + HTTP_REQ_URL, uri, + HTTP_REQ_HEADER, "Accept: application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", + HTTP_REQ_HEADER, user_agent, + HTTP_REQ_OUTFILE, result, + HTTP_REQ_RESULT_STATUS, &status_code, + HTTP_RES_HEADER_CALLBACK, fetch_handle_header, (void*)fd, + NULL, + }; + printf( "GET %s\n", uri ); + http_client_do( request ); + printf( "GET %s -> %ld\n", uri, status_code ); + + fclose(result); + if( status_code != 200 ) { - printf( "Retrying request with HTTP Signature header...\n" ); + unlink( result_filename ); + } - // Load crypto keys - struct crypto_keys* keys = crypto_keys_new(); - if( !crypto_keys_load_private( keys, "data/owner/private.pem" ) ) { - printf( "Failed to load private key\n" ); - return false; - } + return status_code == 200; +} +static bool do_fetch_signed_clearnet( const char* uri, struct fetch_data* fd, const char* result_filename ) +{ + FILE* result = fopen(result_filename,"w"); + if( !result ) { return false; } + + char user_agent[512]; + snprintf( user_agent, sizeof(user_agent), "User-Agent: curl (Apogee/0.1; +https://%s/owner/actor)", g_server->domain ); - // TODO: do signed fetch - struct http_signature hs; - memset( &hs, 0, sizeof(hs) ); - hs.input.method = "get"; - hs.input.url = uri; + // Load crypto keys + struct crypto_keys* keys = crypto_keys_new(); + if( !crypto_keys_load_private( keys, "data/owner/private.pem" ) ) { + printf( "Failed to load private key\n" ); + return false; + } - if( !http_signature_make( keys, &hs ) ) { - return false; - } + // TODO: do signed fetch + struct http_signature hs; + memset( &hs, 0, sizeof(hs) ); + hs.input.method = "get"; + hs.input.url = uri; + + if( !http_signature_make( keys, &hs ) ) { + return false; + } - char date_header[512]; - snprintf( date_header, sizeof(date_header), "Date: %s", hs.date ); - printf( "date_header = %s\n", date_header ); - - char sign_header[512]; - snprintf( sign_header, sizeof(sign_header), "Signature: keyId=\"https://%s/owner/actor#mainKey\",headers=\"(request-target) host date content-length digest\",signature=\"%s\"", - g_server->domain, - hs.signature - ); - printf( "sign_header = %s\n", sign_header ); - - char digest_header[512]; - snprintf( digest_header, sizeof(digest_header), "Digest: %s", hs.digest ); - printf( "digest_header = %s\n", digest_header ); - - char content_length_header[512]; - snprintf( content_length_header, sizeof(content_length_header), "Content-Length: %d", hs.content_length ); - printf( "content_length_header = %s\n", content_length_header ); - - fseek( result, 0, SEEK_SET ); - - const void* request2[] = { - HTTP_REQ_URL, uri, - HTTP_REQ_HEADER, user_agent, - HTTP_REQ_HEADER, date_header, - HTTP_REQ_HEADER, sign_header, - HTTP_REQ_HEADER, content_length_header, - HTTP_REQ_HEADER, digest_header, - HTTP_REQ_HEADER, "Accept: application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", - HTTP_REQ_OUTFILE, result, - HTTP_REQ_RESULT_STATUS, &status_code, - NULL, - }; - - if( !http_client_do( request2 ) ) { - printf( "Failed to perform get, status_code = %ld\n", status_code ); + char date_header[512]; + snprintf( date_header, sizeof(date_header), "Date: %s", hs.date ); + printf( "date_header = %s\n", date_header ); + + char sign_header[512]; + snprintf( sign_header, sizeof(sign_header), "Signature: keyId=\"https://%s/owner/actor#mainKey\",headers=\"(request-target) host date content-length digest\",signature=\"%s\"", + g_server->domain, + hs.signature + ); + printf( "sign_header = %s\n", sign_header ); + + char digest_header[512]; + snprintf( digest_header, sizeof(digest_header), "Digest: %s", hs.digest ); + printf( "digest_header = %s\n", digest_header ); + + char content_length_header[512]; + snprintf( content_length_header, sizeof(content_length_header), "Content-Length: %d", hs.content_length ); + printf( "content_length_header = %s\n", content_length_header ); + + long status_code = 0; + const void* request[] = { + HTTP_REQ_URL, uri, + HTTP_REQ_HEADER, user_agent, + HTTP_REQ_HEADER, date_header, + HTTP_REQ_HEADER, sign_header, + HTTP_REQ_HEADER, content_length_header, + HTTP_REQ_HEADER, digest_header, + HTTP_REQ_HEADER, "Accept: application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", + HTTP_REQ_OUTFILE, result, + HTTP_REQ_RESULT_STATUS, &status_code, + NULL, + }; + + if( !http_client_do( request ) ) { + printf( "Failed to perform get, status_code = %ld\n", status_code ); + } + printf( "GET %s -> %ld (first signed)\n", uri, status_code ); + fclose(result); + + if( status_code == 200 ) { return true; } + + result = fopen(result_filename,"w"); + + const void* request2[] = { + HTTP_REQ_URL, uri, + HTTP_REQ_HEADER, user_agent, + HTTP_REQ_HEADER, date_header, + HTTP_REQ_HEADER, sign_header, + HTTP_REQ_HEADER, content_length_header, + HTTP_REQ_HEADER, digest_header, + HTTP_REQ_HEADER, "Accept: application/activity+json", + HTTP_REQ_OUTFILE, result, + HTTP_REQ_RESULT_STATUS, &status_code, + HTTP_REQ_TIMEOUT, (void*)10, + NULL, + }; + + if( !http_client_do( request2 ) ) { + printf( "Failed to perform get, status_code = %ld\n", status_code ); + } + printf( "GET %s -> %ld (second signed)\n", uri, status_code ); + fclose(result); + + if( status_code == 200 ) { return true; } + + unlink(result_filename); + return false; +} + +static bool do_fetch( const char* uri, const char* fh ) +{ + // Setup fetch data + struct fetch_data fd; + memset(&fd,0,sizeof(fd)); + { + char host_domain[512]; + if( url_get_domain( uri, host_domain, sizeof(host_domain) ) ) { + fd.p = peer_from_domain(host_domain); } - printf( "GET %s -> %ld\n", uri, status_code ); } - return ( status_code == 200 ); + bool result = false; + + if( do_fetch_tor( uri, &fd, fh ) ) { + goto success; + } + if( do_fetch_clearnet( uri, &fd, fh ) ) { + goto success; + } + if( do_fetch_signed_clearnet( uri, &fd, fh ) ) { + goto success; + } + + goto failed; +cleanup: + // Save peer data + if( fd.p ) { + peer_save( fd.p ); + peer_free( fd.p ); + } + return result; +failed: + result = false; + goto cleanup; +success: + result = true; + goto cleanup; } bool pull_remote_file( const char* filename, const char* uri ) { printf( "* Fetching %s\n", uri ); char tmp_filename[512]; - FILE* f = fopen(format(tmp_filename,512,"%s.tmp",filename),"w"); + snprintf( tmp_filename,512, "%s.tmp", filename ); bool result = false; - if( do_fetch( uri, f ) ) { + if( do_fetch( uri, tmp_filename ) ) { rename(tmp_filename,filename); result = true; - fclose(f); - } else { - fclose(f); - // Dump file contents to prevent holding large amounts of disk space inadvertantly - truncate(tmp_filename,0); } return result; diff --git a/src/model/gc.c b/src/model/gc.c index 567b931..8c9202b 100644 --- a/src/model/gc.c +++ b/src/model/gc.c @@ -178,7 +178,9 @@ static void sweep_timelines() void gc_run() { + status_gc(); sweep_posts(); //sweep_timelines(); + } diff --git a/src/model/peer.c b/src/model/peer.c index 2aa3eaa..5f82898 100644 --- a/src/model/peer.c +++ b/src/model/peer.c @@ -23,6 +23,7 @@ static struct json_object_field peer_layout[] = { JSON_FIELD_DATETIME( last_tor_delivery, false ), JSON_FIELD_BOOL( admin_disable_outbox, false ), + JSON_FIELD_BOOL( admin_disable_tor, false ), JSON_FIELD_END, }; diff --git a/src/model/peer.h b/src/model/peer.h index 8d4cf8f..b6732a8 100644 --- a/src/model/peer.h +++ b/src/model/peer.h @@ -12,6 +12,7 @@ struct peer time_t last_successful_delivery; time_t last_failed_delivery; bool admin_disable_outbox; + bool admin_disable_tor; time_t last_hidden_service_delivery; time_t last_tor_delivery; diff --git a/src/model/status.c b/src/model/status.c index 98a171f..8bd24ce 100644 --- a/src/model/status.c +++ b/src/model/status.c @@ -16,7 +16,7 @@ #include "json/json.h" #include "json/layout.h" #include "ffdb/fs_list.h" -#include "ffdb/hash_index.h" +//#include "ffdb/hash_index.h" #include "ffdb/trie.h" #include "sha256/sha256.h" #include "collections/array.h" @@ -216,7 +216,6 @@ struct status* status_from_uri( const char* uri ) struct status* s = status_from_local_uri( uri ); if( s ) { return s; } - int id = -1; char* id_str = ffdb_trie_get( "data/statuses/by-uri", uri ); if( id_str ) { struct status* s = status_from_id(atoi(id_str)); @@ -226,6 +225,10 @@ struct status* status_from_uri( const char* uri ) free(id_str); } + return NULL; + + /* + int id = -1; if( !hash_index_get( "data/statuses/uri", uri, &id ) ) { return NULL; } s = status_from_id(id); @@ -239,6 +242,7 @@ struct status* status_from_uri( const char* uri ) } return s; + */ } void status_add_reply( struct status* s, struct status* child ) { @@ -352,10 +356,11 @@ bool status_sync_from_activity_pub( struct status* s, struct ap_object* act ) status_save_new(s); if( act->in_reply_to ) { - //printf( "Status %d is reply to %s\n", s->id, act->in_reply_to ); + printf( "Status %d is reply to %s\n", s->id, act->in_reply_to ); int parent_id = 0; struct status* parent = status_from_uri_or_fetch( act->in_reply_to ); + //struct status* parent = status_from_uri_or_stub( act->in_reply_to ); if( parent ) { parent_id = parent->id; status_save(parent); @@ -515,10 +520,11 @@ struct status* status_from_uri_or_fetch( const char* uri ) if( s ) { return s; } return status_fetch_from_uri( uri ); } -struct status* status_fetch_from_uri( const char* uri ) +struct status* status_from_uri_or_stub( const char* uri ) { struct status* s = status_from_uri(uri); if( !s ) { + printf( "Creating stub for %s\n", uri ); s = malloc(sizeof(*s)); memset(s,0,sizeof(*s)); @@ -530,12 +536,17 @@ struct status* status_fetch_from_uri( const char* uri ) s->url = strdup(uri); status_save_new(s); - //hash_index_set( "data/statuses/uri", uri, s->id ); char id_str[32]; snprintf( id_str,32, "%d", s->id ); ffdb_trie_set( "data/statuses/by-uri", s->url, id_str ); } + return s; +} +struct status* status_fetch_from_uri( const char* uri ) +{ + struct status* s = status_from_uri_or_stub( uri ); + if( !status_sync_from_uri(s,uri) ) { status_free(s); return NULL; @@ -645,10 +656,10 @@ void status_save( struct status* s ) // Index the status if( s->url ) { - //hash_index_set( "data/statuses/uri", s->url, s->id ); char id_str[32]; snprintf( id_str,32, "%d", s->id ); ffdb_trie_set( "data/statuses/by-uri", s->url, id_str ); + //hash_index_remove( "data/statuses/uri", s->url ); } if( s->stub ) { @@ -1112,3 +1123,37 @@ void status_get_bookmarks( int offset, int limit, void* results_ptr ) free( keys.items ); } +void status_gc() +{ + /* + printf( "Sweeping legacy uri->status_id index\n" ); + // Sweep legacy url to status id index + for( int i = 0; i <= 0xFFFFF; ++i ) { + int offset = 0; + next: + char* key = ffdb_hash_get_key( "data/statuses/uri", i, offset ); + if( !key ) { continue; } + + printf( "Processing key %s\n", key ); + + int id = 0; + if( !hash_index_get( "data/statuses/uri", key, &id ) ) { continue; } + + struct status* s = status_from_id( id ); + if( !s ) { + printf( "Unable to load status %s (id=%d), removing from index.\n", key, id ); + hash_index_remove( "data/statuses/uri", key ); + free(key); + goto next; + } + + // Status still exists, goto next entry + status_save(s); + status_free(s); + free(key); + offset += 1; + goto next; + } + */ +} + diff --git a/src/model/status.h b/src/model/status.h index a76863b..22cbced 100644 --- a/src/model/status.h +++ b/src/model/status.h @@ -103,6 +103,8 @@ struct status* status_new_repost( struct status* s, struct account* a ); struct status* status_from_uri( const char* uri ); struct status* status_fetch_from_uri( const char* uri ); struct status* status_from_uri_or_fetch( const char* uri ); +struct status* status_from_uri_or_stub( const char* uri ); +void status_gc(); struct ap_object; struct status* status_from_activity( struct ap_object* act ); diff --git a/src/view/litepub-0.1.jsonld.template b/src/view/litepub-0.1.jsonld.template new file mode 100644 index 0000000..cdb25ae --- /dev/null +++ b/src/view/litepub-0.1.jsonld.template @@ -0,0 +1,71 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "Emoji": "toot:Emoji", + "Hashtag": "as:Hashtag", + "PropertyValue": "schema:PropertyValue", + "atomUri": "ostatus:atomUri", + "conversation": { + "@id": "ostatus:conversation", + "@type": "@id" + }, + "discoverable": "toot:discoverable", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "capabilities": "litepub:capabilities", + "ostatus": "http://ostatus.org#", + "schema": "http://schema.org#", + "toot": "http://joinmastodon.org/ns#", + "fedibird": "http://fedibird.com/ns#", + "value": "schema:value", + "sensitive": "as:sensitive", + "litepub": "http://litepub.social/ns#", + "invisible": "litepub:invisible", + "directMessage": "litepub:directMessage", + "listMessage": { + "@id": "litepub:listMessage", + "@type": "@id" + }, + "quoteUrl": "as:quoteUrl", + "quoteUri": "fedibird:quoteUri", + "oauthRegistrationEndpoint": { + "@id": "litepub:oauthRegistrationEndpoint", + "@type": "@id" + }, + "EmojiReact": "litepub:EmojiReact", + "ChatMessage": "litepub:ChatMessage", + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" + }, + "vcard": "http://www.w3.org/2006/vcard/ns#", + "sm": "http://smithereen.software/ns#", + "nonAnonymous": "sm:nonAnonymous", + "formerRepresentations": "litepub:formerRepresentations", + "votersCount": "toot:votersCount", + "mz": "https://joinmobilizon.org/ns#", + "joinMode": { + "@id": "mz:joinMode", + "@type": "mz:joinModeType" + }, + "joinModeType": { + "@id": "mz:joinModeType", + "@type": "rdfs:Class" + }, + "participationMessage": { + "@id": "mz:participationMessage", + "@type": "sc:Text" + }, + "streetAddress": "sc:streetAddress", + "postalCode": "sc:postalCode", + "addressLocality": "sc:addressLocality", + "addressRegion": "sc:addressRegion", + "addressCountry": "sc:addressCountry", + "location": { + "@id": "sc:location", + "@type": "sc:Place" + } + } + ] +}