Add build file (was excluded by .gitignore), update .gitignore, add automatic account sync (3-5 day interval), fix inconsistent actor/attributedTo data, move rfc3339 timestamp code to util submodule

.gitignore

@@ -9,6 +9,7 @@ data/
 assets/soapbox*
 backup/
 vgcore.*
-apogee*
+apogee
+apogee.debug
 /*.log
 /*.json

src/ap

@@ -1 +1 @@
-Subproject commit 64797a74e8955aa500deaff1e6883b7349022892
+Subproject commit 67471e0105e63d7fb407c89af0f0dbecf12695c2

src/apogee.build.json

@@ -0,0 +1,35 @@
+{
+	"cxx":{
+		"cc": "gcc",
+		"ld": "gcc",
+		"ar": "ar",
+		"flags": "-g -MP -MD -Os -Og -Werror -flto -Wall -I./ -Isrc/",
+		"ldflags": "-lcurl -lssl -lcrypto -flto -Os",
+		"output": "obj/apogee",
+		"objdir": "obj/apogee/"
+	},
+	"tools": [ "cxx" ],
+	"sources": [
+		"src/ap",
+		"src/collections",
+		"src/controller",
+		"src/ffdb",
+		"src/http",
+		"src/json",
+		"src/json-ld",
+		"src/model.c",
+		"src/model",
+		"src/rdf",
+		"src/reflect",
+		"src/rsa-signature-2017",
+		"src/sha256/sha256.c",
+		"src/util",
+		"src/view",
+
+		"src/dev.c",
+		"src/form.c",
+		"src/tor.c",
+		"src/process.c",
+		"src/main.c"
+	]
+}

src/controller/fetch.c

@@ -2,6 +2,7 @@
 
 // Model
 #include "model/status.h"
+#include "model/account.h"
 
 // Submodules
 #include "ffdb/trie.h"
@@ -59,10 +60,27 @@ static void process_fetch_once()
 	printf( "new count = %d\n", ffdb_trie_count(STUBS) );
 }
 
+static void process_account_sync_once()
+{
+	for( int i = 0; i < 10; ++i ) {
+		struct account* a = account_load_next_update();
+		if( !a ) { continue; }
+
+		if( time(NULL) >= a->next_update ) {
+			account_sync_from_activity_pub( a->id );
+		}
+
+		account_free(a);
+
+		return;
+	}
+}
+
 void process_fetch()
 {
 	while(true) {
 		process_fetch_once();
+		process_account_sync_once();
 
 		for( int i = 0; i < 30; ++i ) {
 			if( terminate ) { return; };

src/ffdb

@@ -1 +1 @@
-Subproject commit 7e76a21254e772b6bcc955fcb575b5f40a67dbb4
+Subproject commit 06066e9de27b265e1191de272a67e5e4ba2353f8

src/model/account.c

@@ -1,16 +1,6 @@
 #define _GNU_SOURCE
 #include "account.h"
 
-// Submodules
-#include "json/json.h"
-#include "json/layout.h"
-#include "ffdb/fs_list.h"
-#include "ffdb/hash_index.h"
-#include "ffdb/trie.h"
-#include "util/format.h"
-#include "collections/array.h"
-#include "ap/object.h"
-
 // Model
 #include "model/server.h"
 #include "model/status.h"
@@ -24,6 +14,17 @@
 // View
 #include "view/api/Relationship.h"
 
+// Submodules
+#include "json/json.h"
+#include "json/layout.h"
+#include "ffdb/fs_list.h"
+#include "ffdb/hash_index.h"
+#include "ffdb/trie.h"
+#include "collections/array.h"
+#include "ap/object.h"
+#include "util/format.h"
+#include "util/time.h"
+
 // Stdlib
 #include <stdio.h>
 #include <stdlib.h>
@@ -65,6 +66,7 @@ static struct json_object_field account_layout[] = {
 	JSON_FIELD_STRING( display_name, true ),
 	JSON_FIELD_BOOL( stub, false ),
 	JSON_FIELD_DATETIME( next_stub_recheck, false ),
+	JSON_FIELD_DATETIME( next_update, false ),
 	JSON_FIELD_STRING( avatar, false ),
 	{
 		.key = "avatar",
@@ -450,12 +452,75 @@ void account_save( struct account* a )
 
 	json_write_object_layout_to_file( filename, "\t", account_layout, a );
 
+	// Index by uri
 	if( a->account_url ) {
 		char id_str[32];
 		snprintf( id_str,32, "%d", a->id );
 		ffdb_trie_set( "data/accounts/by-uri", a->account_url, id_str );
 		hash_index_remove( "data/accounts/uri_index/", a->account_url );
 	}
+
+	// Index by next_update
+	{
+		char timestamp[128];
+		rfc3339_time_string( a->next_update, timestamp, sizeof(timestamp) );
+		char key[512];
+		snprintf( key,512, "%s-%d", timestamp, a->id );
+		char id_str[32];
+		snprintf( id_str,32, "%d", a->id );
+		ffdb_trie_set( "data/accounts/updates/", key, id_str );
+	}
+}
+struct account* account_load_next_update()
+{
+	int count = ffdb_trie_count( "data/accounts/updates/" );
+	if( count == 0 ) {
+		printf( "No accounts to update\n" );
+		return NULL;
+	}
+
+	char* key = NULL;
+	char* value = NULL;
+	count -= 1;
+	while( !ffdb_trie_get_index( "data/accounts/updates/", count, &key, &value ) ) {
+		if( !count ) {
+			return NULL;
+		}
+
+		printf( "Unable to load index entry #%d\n", count );
+		free(key); key = NULL;
+		free(value); value = NULL;
+		count -= 1;
+	}
+
+	struct account* result = account_from_id( atoi(value) );
+	if( !result ) {
+		printf( "Unable to load account with id=%s\n", value );
+		goto reindex;
+	}
+
+	// Check to make sure the index entry is valid
+	char timestamp[512];
+	rfc3339_time_string( result->next_update, timestamp, 512 );
+	char buffer[512];
+	snprintf( buffer,sizeof(buffer), "%s-%s", timestamp, value );
+	if( 0 != strcmp( buffer, key ) ) {
+		printf( "Mismatch between index and account data (%s != %s). Fixing\n", key, buffer );
+		goto reindex;
+	}
+
+	free(key);
+	free(value);
+
+	printf( "Next account to update: %s\n", result->account_url );
+	return result;
+reindex:
+	// It isn't, fix the index and retry
+	ffdb_trie_set( "data/accounts/updates/", key, NULL );
+	account_free(result);
+	free(key);
+	free(value);
+	return NULL;
 }
 
 

src/model/account.h

@@ -44,6 +44,7 @@ struct account
 	bool stub;
 	bool defunct;
 	time_t next_stub_recheck;
+	time_t next_update;
 	int replaced_by;
 
 	int account_type;
@@ -100,6 +101,7 @@ struct account* account_from_webfinger( const char* handle, const char* default_
 struct account* account_new();
 struct account* account_from_uri_or_fetch( const char* uri );
 struct account* account_fetch_from_uri( const char* uri );
+struct account* account_load_next_update();
 void account_free( struct account* a );
 void account_save( struct account* a );
 

src/model/account/ap_sync.c

@@ -36,14 +36,19 @@ bool account_sync_from_activity_pub( unsigned int account_id )
 {
 	bool result = false;
 	struct account* a = account_from_id(account_id);
+	struct ap_object* obj = NULL;
 	if( !a ) {
 		a = account_new();
 		a->id = account_id;
+	} else if( a->next_update > time(NULL) ) {
+		goto succeeded;
+	} else if( a->local ) {
+		goto succeeded;
 	}
 
-	struct ap_object* obj = account_get_activity_pub_data(a);
+	obj = account_get_activity_pub_data(a);
 	if( !obj ) {
-		printf( "Failed to load AP data\n" );
+		printf( "Failed to load AP data for %s (%d)\n", a->account_url, a->id );
 		goto failed;
 	}
 
@@ -51,6 +56,7 @@ bool account_sync_from_activity_pub( unsigned int account_id )
 
 	// If we got here, this account can't be a stub any more
 	a->stub = false;
+	a->next_update = time(NULL) + (60*60*24*3) + (rand() % (60*60*24*2)); // Next update in 3-5 days
 
 	account_save(a);
 	account_index_webfinger(a);
@@ -60,6 +66,10 @@ succeeded:
 	result = true;
 	goto cleanup;
 cleanup:
+	if( a ) {
+		a->next_update = time(NULL) + (60*60*24*3) + (rand() % (60*60*24*2)); // Next update in 3-5 days
+		account_save(a);
+	}
 	ap_object_free(obj);
 	account_free(a);
 

src/model/activity.c

@@ -99,7 +99,7 @@ struct ap_object* activity_create_EmojiReact( struct status* s, const char* reac
 	struct ap_object* act = activity_new_local_activity();
 	activity_allocate_local_id(act);
 	act->id = aformat( "https://%s/activity/%d", g_server->domain, act->local_id );
-	act->actor = aformat( "https://%s/owner/actor", g_server->domain );
+	act->actor = strdup(a->account_url);
 	act->type = pleroma_EmojiReact;
 	act->content.content = safe_strdup(react);
 	act->published = time(NULL);
@@ -221,16 +221,17 @@ failed:
 
 struct ap_object* activity_create_Like( struct status* s )
 {
+	struct account* a = account_from_id( s->account_id );
+
 	struct ap_object* act = activity_new_local_activity();
 	activity_allocate_local_id(act);
 	act->id = aformat( "https://%s/activity/%d", g_server->domain, act->local_id );
-	act->actor = aformat( "https://%s/owner/actor", g_server->domain );
+	act->actor = strdup(a->account_url);
 	act->type = ap_Like;
 	act->published = time(NULL);
 	act->object.tag = apaot_ref;
 	act->object.ref = strdup( s->url );
 
-	struct account* a = account_from_id( s->account_id );
 	char* to = aformat( "https://%s/owner/followers", g_server->domain );
 	array_append( &act->to, sizeof(to), &to );
 	to = strdup(a->account_url);

src/model/timeline.c

@@ -1,12 +1,16 @@
 #include "timeline.h"
 
+// Model
+#include "model/status.h"
+
+// Submodules
 #include "json/json.h"
 #include "json/layout.h"
 #include "ffdb/trie.h"
 #include "util/format.h"
+#include "util/time.h"
 
-#include "model/status.h"
+// Standard Library
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <stddef.h>
@@ -78,23 +82,10 @@ int timeline_load_statuses( struct timeline* tl, int offset_from_head, int count
 	free(keys.items);
 	return result_count;
 }
-static void key_for_post( struct status* s, char* key, int sizeof_key )
-{
-	struct tm gmtime_data;
-	gmtime_r( &s->published, &gmtime_data );
-	snprintf( key, sizeof_key, "%04d-%02d-%02dT%02d:%02d:%02dZ",
-		gmtime_data.tm_year + 1900,
-		gmtime_data.tm_mon + 1,
-		gmtime_data.tm_mday,
-		gmtime_data.tm_hour,
-		gmtime_data.tm_min,
-		gmtime_data.tm_sec
-	);
-}
 void timeline_add_post( struct timeline* tl, struct status* s )
 {
 	char key[512];
-	key_for_post(s,key,sizeof(key));
+	rfc3339_time_string( s->published, key, sizeof(key) );
 
 	char value[32];
 	ffdb_trie_set( tl->path, key, format( value, sizeof(value), "%d", s->id ) );
@@ -102,7 +93,7 @@ void timeline_add_post( struct timeline* tl, struct status* s )
 void timeline_remove_post( struct timeline* tl, struct status* s )
 {
 	char key[512];
-	key_for_post(s,key,sizeof(key));
+	rfc3339_time_string( s->published, key, sizeof(key) );
 
 	ffdb_trie_remove( tl->path, key );
 }

src/util

@@ -1 +1 @@
-Subproject commit 24b374ef03befcff66499be372d089ade16e2381
+Subproject commit e133427bb7da0224fc365252406024c207418863