From 5b04add122d712682387da88e69a07092f5a744e Mon Sep 17 00:00:00 2001 From: teknomunk Date: Fri, 21 Jul 2023 19:05:27 -0500 Subject: [PATCH] Add build file (was excluded by .gitignore), update .gitignore, add automatic account sync (3-5 day interval), fix inconsistent actor/attributedTo data, move rfc3339 timestamp code to util submodule --- .gitignore | 3 +- src/ap | 2 +- src/apogee.build.json | 35 +++++++++++++++ src/controller/fetch.c | 18 ++++++++ src/ffdb | 2 +- src/model/account.c | 85 ++++++++++++++++++++++++++++++++----- src/model/account.h | 2 + src/model/account/ap_sync.c | 14 +++++- src/model/activity.c | 7 +-- src/model/timeline.c | 25 ++++------- src/util | 2 +- 11 files changed, 159 insertions(+), 36 deletions(-) create mode 100644 src/apogee.build.json diff --git a/.gitignore b/.gitignore index b75d110..3887c73 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ data/ assets/soapbox* backup/ vgcore.* -apogee* +apogee +apogee.debug /*.log /*.json diff --git a/src/ap b/src/ap index 64797a7..67471e0 160000 --- a/src/ap +++ b/src/ap @@ -1 +1 @@ -Subproject commit 64797a74e8955aa500deaff1e6883b7349022892 +Subproject commit 67471e0105e63d7fb407c89af0f0dbecf12695c2 diff --git a/src/apogee.build.json b/src/apogee.build.json new file mode 100644 index 0000000..78f051c --- /dev/null +++ b/src/apogee.build.json @@ -0,0 +1,35 @@ +{ + "cxx":{ + "cc": "gcc", + "ld": "gcc", + "ar": "ar", + "flags": "-g -MP -MD -Os -Og -Werror -flto -Wall -I./ -Isrc/", + "ldflags": "-lcurl -lssl -lcrypto -flto -Os", + "output": "obj/apogee", + "objdir": "obj/apogee/" + }, + "tools": [ "cxx" ], + "sources": [ + "src/ap", + "src/collections", + "src/controller", + "src/ffdb", + "src/http", + "src/json", + "src/json-ld", + "src/model.c", + "src/model", + "src/rdf", + "src/reflect", + "src/rsa-signature-2017", + "src/sha256/sha256.c", + "src/util", + "src/view", + + "src/dev.c", + "src/form.c", + "src/tor.c", + "src/process.c", + "src/main.c" + ] +} diff --git a/src/controller/fetch.c b/src/controller/fetch.c index 2266912..a9bbd53 100644 --- a/src/controller/fetch.c +++ b/src/controller/fetch.c @@ -2,6 +2,7 @@ // Model #include "model/status.h" +#include "model/account.h" // Submodules #include "ffdb/trie.h" @@ -59,10 +60,27 @@ static void process_fetch_once() printf( "new count = %d\n", ffdb_trie_count(STUBS) ); } +static void process_account_sync_once() +{ + for( int i = 0; i < 10; ++i ) { + struct account* a = account_load_next_update(); + if( !a ) { continue; } + + if( time(NULL) >= a->next_update ) { + account_sync_from_activity_pub( a->id ); + } + + account_free(a); + + return; + } +} + void process_fetch() { while(true) { process_fetch_once(); + process_account_sync_once(); for( int i = 0; i < 30; ++i ) { if( terminate ) { return; }; diff --git a/src/ffdb b/src/ffdb index 7e76a21..06066e9 160000 --- a/src/ffdb +++ b/src/ffdb @@ -1 +1 @@ -Subproject commit 7e76a21254e772b6bcc955fcb575b5f40a67dbb4 +Subproject commit 06066e9de27b265e1191de272a67e5e4ba2353f8 diff --git a/src/model/account.c b/src/model/account.c index c9f3900..5d19822 100644 --- a/src/model/account.c +++ b/src/model/account.c @@ -1,16 +1,6 @@ #define _GNU_SOURCE #include "account.h" -// Submodules -#include "json/json.h" -#include "json/layout.h" -#include "ffdb/fs_list.h" -#include "ffdb/hash_index.h" -#include "ffdb/trie.h" -#include "util/format.h" -#include "collections/array.h" -#include "ap/object.h" - // Model #include "model/server.h" #include "model/status.h" @@ -24,6 +14,17 @@ // View #include "view/api/Relationship.h" +// Submodules +#include "json/json.h" +#include "json/layout.h" +#include "ffdb/fs_list.h" +#include "ffdb/hash_index.h" +#include "ffdb/trie.h" +#include "collections/array.h" +#include "ap/object.h" +#include "util/format.h" +#include "util/time.h" + // Stdlib #include #include @@ -65,6 +66,7 @@ static struct json_object_field account_layout[] = { JSON_FIELD_STRING( display_name, true ), JSON_FIELD_BOOL( stub, false ), JSON_FIELD_DATETIME( next_stub_recheck, false ), + JSON_FIELD_DATETIME( next_update, false ), JSON_FIELD_STRING( avatar, false ), { .key = "avatar", @@ -450,12 +452,75 @@ void account_save( struct account* a ) json_write_object_layout_to_file( filename, "\t", account_layout, a ); + // Index by uri if( a->account_url ) { char id_str[32]; snprintf( id_str,32, "%d", a->id ); ffdb_trie_set( "data/accounts/by-uri", a->account_url, id_str ); hash_index_remove( "data/accounts/uri_index/", a->account_url ); } + + // Index by next_update + { + char timestamp[128]; + rfc3339_time_string( a->next_update, timestamp, sizeof(timestamp) ); + char key[512]; + snprintf( key,512, "%s-%d", timestamp, a->id ); + char id_str[32]; + snprintf( id_str,32, "%d", a->id ); + ffdb_trie_set( "data/accounts/updates/", key, id_str ); + } +} +struct account* account_load_next_update() +{ + int count = ffdb_trie_count( "data/accounts/updates/" ); + if( count == 0 ) { + printf( "No accounts to update\n" ); + return NULL; + } + + char* key = NULL; + char* value = NULL; + count -= 1; + while( !ffdb_trie_get_index( "data/accounts/updates/", count, &key, &value ) ) { + if( !count ) { + return NULL; + } + + printf( "Unable to load index entry #%d\n", count ); + free(key); key = NULL; + free(value); value = NULL; + count -= 1; + } + + struct account* result = account_from_id( atoi(value) ); + if( !result ) { + printf( "Unable to load account with id=%s\n", value ); + goto reindex; + } + + // Check to make sure the index entry is valid + char timestamp[512]; + rfc3339_time_string( result->next_update, timestamp, 512 ); + char buffer[512]; + snprintf( buffer,sizeof(buffer), "%s-%s", timestamp, value ); + if( 0 != strcmp( buffer, key ) ) { + printf( "Mismatch between index and account data (%s != %s). Fixing\n", key, buffer ); + goto reindex; + } + + free(key); + free(value); + + printf( "Next account to update: %s\n", result->account_url ); + return result; +reindex: + // It isn't, fix the index and retry + ffdb_trie_set( "data/accounts/updates/", key, NULL ); + account_free(result); + free(key); + free(value); + return NULL; } diff --git a/src/model/account.h b/src/model/account.h index af2683e..26ea8cf 100644 --- a/src/model/account.h +++ b/src/model/account.h @@ -44,6 +44,7 @@ struct account bool stub; bool defunct; time_t next_stub_recheck; + time_t next_update; int replaced_by; int account_type; @@ -100,6 +101,7 @@ struct account* account_from_webfinger( const char* handle, const char* default_ struct account* account_new(); struct account* account_from_uri_or_fetch( const char* uri ); struct account* account_fetch_from_uri( const char* uri ); +struct account* account_load_next_update(); void account_free( struct account* a ); void account_save( struct account* a ); diff --git a/src/model/account/ap_sync.c b/src/model/account/ap_sync.c index 3c851b7..cccd701 100644 --- a/src/model/account/ap_sync.c +++ b/src/model/account/ap_sync.c @@ -36,14 +36,19 @@ bool account_sync_from_activity_pub( unsigned int account_id ) { bool result = false; struct account* a = account_from_id(account_id); + struct ap_object* obj = NULL; if( !a ) { a = account_new(); a->id = account_id; + } else if( a->next_update > time(NULL) ) { + goto succeeded; + } else if( a->local ) { + goto succeeded; } - struct ap_object* obj = account_get_activity_pub_data(a); + obj = account_get_activity_pub_data(a); if( !obj ) { - printf( "Failed to load AP data\n" ); + printf( "Failed to load AP data for %s (%d)\n", a->account_url, a->id ); goto failed; } @@ -51,6 +56,7 @@ bool account_sync_from_activity_pub( unsigned int account_id ) // If we got here, this account can't be a stub any more a->stub = false; + a->next_update = time(NULL) + (60*60*24*3) + (rand() % (60*60*24*2)); // Next update in 3-5 days account_save(a); account_index_webfinger(a); @@ -60,6 +66,10 @@ succeeded: result = true; goto cleanup; cleanup: + if( a ) { + a->next_update = time(NULL) + (60*60*24*3) + (rand() % (60*60*24*2)); // Next update in 3-5 days + account_save(a); + } ap_object_free(obj); account_free(a); diff --git a/src/model/activity.c b/src/model/activity.c index 4c3514b..89f7f83 100644 --- a/src/model/activity.c +++ b/src/model/activity.c @@ -99,7 +99,7 @@ struct ap_object* activity_create_EmojiReact( struct status* s, const char* reac struct ap_object* act = activity_new_local_activity(); activity_allocate_local_id(act); act->id = aformat( "https://%s/activity/%d", g_server->domain, act->local_id ); - act->actor = aformat( "https://%s/owner/actor", g_server->domain ); + act->actor = strdup(a->account_url); act->type = pleroma_EmojiReact; act->content.content = safe_strdup(react); act->published = time(NULL); @@ -221,16 +221,17 @@ failed: struct ap_object* activity_create_Like( struct status* s ) { + struct account* a = account_from_id( s->account_id ); + struct ap_object* act = activity_new_local_activity(); activity_allocate_local_id(act); act->id = aformat( "https://%s/activity/%d", g_server->domain, act->local_id ); - act->actor = aformat( "https://%s/owner/actor", g_server->domain ); + act->actor = strdup(a->account_url); act->type = ap_Like; act->published = time(NULL); act->object.tag = apaot_ref; act->object.ref = strdup( s->url ); - struct account* a = account_from_id( s->account_id ); char* to = aformat( "https://%s/owner/followers", g_server->domain ); array_append( &act->to, sizeof(to), &to ); to = strdup(a->account_url); diff --git a/src/model/timeline.c b/src/model/timeline.c index 5c4e7df..db72e94 100644 --- a/src/model/timeline.c +++ b/src/model/timeline.c @@ -1,12 +1,16 @@ #include "timeline.h" +// Model +#include "model/status.h" + +// Submodules #include "json/json.h" #include "json/layout.h" #include "ffdb/trie.h" #include "util/format.h" +#include "util/time.h" -#include "model/status.h" - +// Standard Library #include #include #include @@ -78,23 +82,10 @@ int timeline_load_statuses( struct timeline* tl, int offset_from_head, int count free(keys.items); return result_count; } -static void key_for_post( struct status* s, char* key, int sizeof_key ) -{ - struct tm gmtime_data; - gmtime_r( &s->published, &gmtime_data ); - snprintf( key, sizeof_key, "%04d-%02d-%02dT%02d:%02d:%02dZ", - gmtime_data.tm_year + 1900, - gmtime_data.tm_mon + 1, - gmtime_data.tm_mday, - gmtime_data.tm_hour, - gmtime_data.tm_min, - gmtime_data.tm_sec - ); -} void timeline_add_post( struct timeline* tl, struct status* s ) { char key[512]; - key_for_post(s,key,sizeof(key)); + rfc3339_time_string( s->published, key, sizeof(key) ); char value[32]; ffdb_trie_set( tl->path, key, format( value, sizeof(value), "%d", s->id ) ); @@ -102,7 +93,7 @@ void timeline_add_post( struct timeline* tl, struct status* s ) void timeline_remove_post( struct timeline* tl, struct status* s ) { char key[512]; - key_for_post(s,key,sizeof(key)); + rfc3339_time_string( s->published, key, sizeof(key) ); ffdb_trie_remove( tl->path, key ); } diff --git a/src/util b/src/util index 24b374e..e133427 160000 --- a/src/util +++ b/src/util @@ -1 +1 @@ -Subproject commit 24b374ef03befcff66499be372d089ade16e2381 +Subproject commit e133427bb7da0224fc365252406024c207418863