Add status sync command, get more Activities parsing

src/controller/api/status.c

@@ -29,7 +29,6 @@ void api_status_write_as_json( struct status* s, FILE* f )
 
 	struct status* repost = NULL;
 	if( s->repost_id ) {
-		printf( "s->repost_id = %d\n", s->repost_id );
 		repost = status_from_id( s->repost_id );
 	}
 

src/controller/cli.c

@@ -2,7 +2,9 @@
 
 #include "model/account.h"
 #include "model/status.h"
+#include "model/ap/activity.h"
 #include "controller/inbox.h"
+#include "format.h"
 
 #include <stdio.h>
 #include <stdbool.h>
@@ -91,13 +93,33 @@ static bool handle_command_status_import( struct cli_request* req )
 		status_free(s);
 	}
 }
+static bool handle_command_status_sync( struct cli_request* req )
+{
+	int res = cli_route_command( req, "sync", 1, "[status id]" );
+	if( res != 1 ) { return !!res; }
+
+	int id = 0;
+	if( 1 != sscanf( req->argv[0], "%d", &id ) ) {
+		return true;
+	}
+
+	struct status* s = status_from_id(id);
+	if( !s ) { return true; }
+
+	status_sync(s);
+	status_save(s);
+	status_free(s);
+	return true;
+}
 static bool handle_command_status( struct cli_request* req )
 {
-	int res = cli_route_command( req, "status", 1, "(import) ..." );
+	int res = cli_route_command( req, "status", 1, "(import|sync) ..." );
 	if( res != 1 ) { return !!res; }
 
 	return false
-	|| handle_command_status_import( req );
+	|| handle_command_status_import( req )
+	|| handle_command_status_sync( req )
+	;
 }
 
 void handle_command( char** argv, int argc )

src/controller/inbox.c

@@ -185,7 +185,7 @@ static bool route_emoji_react( struct ap_activity* act )
 	struct account* reactor = lookup_actor_account(act);
 	if( !s || !reactor ) { return false; }
 
-	status_add_react( s, act->content.value, reactor );
+	status_add_react( s, act->content.content, reactor );
 	return true;
 }
 
@@ -282,12 +282,14 @@ static bool route_accept( struct ap_activity* act )
 
 static bool route_announce( struct ap_activity* act )
 {
+	bool result = false;
+	struct status* s = NULL;
 	struct account* owner_account = account_from_id( owner_account_id );
 	struct account* actor_account = account_from_uri( act->actor );
 	if( !account_does_follow( owner_account, actor_account->id ) ) {
 		// Not following, discard
 		printf( "%s does not follow %s\n", owner_account->handle, actor_account->account_url );
-		return true;
+		goto discard;
 	}
 
 	// Reposts do not show up in notifications unless notifications for the user are enabled
@@ -297,13 +299,15 @@ static bool route_announce( struct ap_activity* act )
 
 	if( act->object.tag != apaot_ref ) {
 		printf( "! object not reference\n" );
-		return false;
+		goto failed;
 	}
 
 	struct status* original_post = status_fetch_from_uri( act->object.ref );
+	if( !original_post ) {
+		goto discard;
+	}
 
 	// TODO: handle repost
-	struct status* s;
 	s = malloc(sizeof(*s));
 	memset(s,0,sizeof(*s));
 
@@ -319,10 +323,20 @@ static bool route_announce( struct ap_activity* act )
 	status_add_to_timeline( s, public_timeline_id );
 	status_add_to_timeline( s, actor_account->id );
 
+	result = true;
+cleanup:
 	status_free(s);
 	status_free(original_post);
+	account_free(owner_account);
+	account_free(actor_account);
 
-	return true;
+	return result;
+discard:
+	result = true;
+	goto cleanup;
+failed:
+	result = false;
+	goto cleanup;
 }
 
 bool route_activity( struct ap_activity* act )

src/ffdb

@@ -1 +1 @@
-Subproject commit e8b0c2ccde301586e53f20563718308a1fc51fff
+Subproject commit 730c340d66cd2c6775a95d297ad1c97ff29126c1

src/json

@@ -1 +1 @@
-Subproject commit 7031acbfc47d71d648d647451a9bd90cb5af7c2b
+Subproject commit 68edc18e9b79bd144049c0aaec5cc6fbdfc0db2c

src/model/account.c

@@ -162,7 +162,7 @@ struct account* account_from_uri_or_fetch( const char* uri )
 }
 struct account* account_from_webfinger( const char* handle )
 {
-	printf( "account_from_webfinger( %s )\n", handle );
+	//printf( "account_from_webfinger( %s )\n", handle );
 	int id;
 	if( !hash_index_get( "data/accounts/webfinger", handle, &id ) ) {
 		printf( "! No account id for %s\n", handle );
@@ -267,8 +267,6 @@ struct account* account_fetch_from_uri( const char* uri )
 		index_uri_to_account_id( uri, account_id );
 	}
 
-	printf( "account_id = %d\n", account_id );
-
 	create_account_skeleton(account_id);
 
 	// Fetch the ActivityPub actor data if we don't already have it
@@ -296,6 +294,7 @@ struct account* account_fetch_from_uri( const char* uri )
 			printf( "! Unable to fetch %s\n", uri );
 			return NULL;
 		}
+		fflush(tmp);
 		printf( "status_code = %d\n", status_code );
 
 		rename(tmp_filename,filename);

src/model/ap/activity.c

@@ -17,6 +17,12 @@
 #include <stdio.h>
 #include <string.h>
 
+char* safe_strdup( const char* str )
+{
+	if( !str ) { return NULL; }
+	return strdup(str);
+}
+
 struct ap_activity* ap_activity_new()
 {
 	struct ap_activity* act = malloc(sizeof(struct ap_activity));
@@ -35,9 +41,8 @@ struct ap_activity* ap_activity_dup( struct ap_activity* act )
 	new_act->actor = strdup(act->actor);
 	new_act->published = act->published;
 
-	if( act->source.value ) {
-		new_act->source.value = strdup(act->source.value);
-	}
+	new_act->source.content = safe_strdup(act->source.content);
+	new_act->source.mime_type = safe_strdup(act->source.mime_type);
 
 	array_dup( &new_act->to, sizeof(char*), &act->to );
 	for( int i = 0; i < new_act->to.count; ++i ) {
@@ -64,9 +69,7 @@ struct ap_activity* ap_activity_dup( struct ap_activity* act )
 	}
 	new_act->object.tag = act->object.tag;
 
-	if( act->state ) {
-		new_act->state = strdup( act->state );
-	}
+	new_act->state = safe_strdup( act->state );
 
 	if( act->has_signature ) {
 		new_act->has_signature = true;
@@ -88,8 +91,8 @@ void ap_activity_free_composite( struct ap_activity* act )
 {
 	free(act->id);
 	free(act->actor);
-	free(act->content.value);
-	free(act->source.value);
+	free(act->content.content);
+	free(act->source.content);
 
 	for( int i = 0; i < act->to.count; ++i ) {
 		free(act->to.items[i]);
@@ -166,7 +169,7 @@ struct ap_activity* ap_activity_create_emoji_react( struct status* s, const char
 	act->id = aformat( "https://%s/activity/%d", g_server_name, id );
 	act->actor = aformat( "https://%s/owner/actor", g_server_name );
 	act->type = apat_emoji_react;
-	act->content.value = strdup(react);
+	act->content.content = safe_strdup(react);
 	act->published = time(NULL);
 	act->object.tag = apaot_ref;
 	act->object.ref = strdup( s->url );

src/model/ap/activity.h

@@ -7,6 +7,14 @@
 
 #include <time.h>
 
+struct ap_activity_source
+{
+	char* mime_type;
+	char* content;
+
+	// TODO: add map;
+};
+
 enum ap_signature_type
 {
 	apst_rsa_signature_2017 = 1,
@@ -57,6 +65,16 @@ enum ap_activity_object_type {
 	apaot_activity = 2,
 };
 
+struct ap_attachement
+{
+	int type;
+	char* mediaType;
+	char* name;
+	char* url;
+};
+void ap_attachement_free( struct ap_attachement* a );
+extern struct json_field_type ap_attachement_type;
+
 struct ap_activity
 {
 	struct ap_activity_context ap_context;
@@ -73,17 +91,11 @@ struct ap_activity
 
 	char* attributed_to;
 
-	struct {
-		char* value;
-		char* mime_type;
-	} content;
+	struct ap_activity_source content;
 
 	char* conversation;
 	time_t published;
-	struct {
-		char* value;
-		char* mime_type;
-	} source;
+	struct ap_activity_source source;
 
 	char* summary;
 
@@ -95,7 +107,7 @@ struct ap_activity
 	} tags;
 
 	struct {
-		char** items;
+		struct ap_attachement** items;
 		int count;
 	} attachments;
 

src/model/ap/activity/attachement.c

@@ -0,0 +1,24 @@
+#include "../activity.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#define OBJ_TYPE struct ap_attachement
+struct json_object_field ap_attachement_layout[] = {
+	JSON_FIELD_STRING( mediaType, false ),
+	JSON_FIELD_STRING( name, false ),
+	JSON_FIELD_STRING( url, true ),
+	JSON_FIELD_END
+};
+#undef OBJ_TYPE
+void ap_attachement_free( struct ap_attachement* a )
+{
+	if( !a ) { return; }
+
+	free(a->mediaType);
+	free(a->name);
+	free(a->url);
+	free(a);
+}
+JSON_FIELD_TYPE_OBJECT_LAYOUT_WITH_DEFAULTS( ap_attachement );
+

src/model/ap/activity/context.c

@@ -59,6 +59,7 @@ static bool context_reader( struct json_pull_parser* jpp, void* field_data, stru
 			static const char* langs[] = {
 				"und",
 				"en",
+				"de",
 			};
 			ctx->language = clang_unknown;
 			for( int i = 0; i < sizeof(langs)/sizeof(langs[0]); ++i ) {

src/model/ap/activity/context.h

@@ -5,6 +5,7 @@ enum context_language
 	clang_unspecified = 0,
 	clang_undefined = 1,
 	clang_en = 2,
+	clang_de = 3,
 	clang_unknown = 999999,
 };
 

src/model/ap/activity/layout.c

@@ -7,18 +7,8 @@
 #include <string.h>
 #include <stddef.h>
 
-static struct json_enum ap_signature_type_enum[] = {
-	{ "RsaSignature2017", 1 },
-	{ NULL, 0 },
-};
-
-static struct json_object_field ap_signature_layout[] = {
-	{ "type",			offsetof( struct ap_signature, type ),		true,  &json_field_enum, ap_signature_type_enum },
-	{ "creator",		offsetof( struct ap_signature, creator ),	true,  &json_field_string },
-	{ "created",		offsetof( struct ap_signature, created ),	true,  &json_field_date_time },
-	{ "signatureValue", offsetof( struct ap_signature, value ),		true,  &json_field_string },
-	{ NULL },
-};
+extern struct json_object_field ap_signature_layout[];
+extern struct json_field_type ap_activity_source_type;
 
 // https://www.w3.org/TR/activitystreams-vocabulary/#h-activity-types
 struct json_enum ap_activity_type_enum[] = {
@@ -56,16 +46,14 @@ struct json_enum ap_activity_type_enum[] = {
 
 struct json_object_field activity_ref_types[] = {
 	{ (char*)apaot_ref,			offsetof( struct ap_activity, object.ref ),	false,	&json_field_string },
-	{ (char*)apaot_activity,		offsetof( struct ap_activity, object.ptr ),	false,	&json_field_object_pointer, ap_activity_layout },
-	{ NULL },
+	{ (char*)apaot_activity,		offsetof( struct ap_activity, object.ptr ),	false,	&ap_activity_type },
+	JSON_FIELD_END,
 };
 struct json_object_field signature_types[] = {
 	{ (char*)1,		offsetof( struct ap_activity, signature ),	false,  &json_field_object_composite, ap_signature_layout },
-	{ NULL },
+	JSON_FIELD_END
 };
 
-
-
 #define OBJ_TYPE struct ap_activity
 struct json_object_field ap_activity_layout[] = {
 	{
@@ -81,9 +69,9 @@ struct json_object_field ap_activity_layout[] = {
 
 	{
 		.key = "content",
-		.offset = offsetof( struct ap_activity, content.value ),
+		.offset = offsetof( struct ap_activity, content ),
 		.required = false,
-		.type = &json_field_string
+		.type = &ap_activity_source_type
 	},
 	JSON_FIELD_STRING(conversation,false),
 	JSON_FIELD_DATETIME(published,false),
@@ -95,9 +83,9 @@ struct json_object_field ap_activity_layout[] = {
 	},
 	{
 		.key = "source",
-		.offset = offsetof( struct ap_activity, source.value ),
+		.offset = offsetof( struct ap_activity, source ),
 		.required = false,
-		.type = &json_field_string
+		.type = &ap_activity_source_type
 	},
 	JSON_FIELD_STRING(summary,false),
 
@@ -117,9 +105,14 @@ struct json_object_field ap_activity_layout[] = {
 		.type = &json_field_string,
 	},
 
-	{ "directMessage",	offsetof( struct ap_activity, direct_message ),	false, &json_field_bool },
+	{
+		.key = "directMessage",
+		.offset = offsetof( struct ap_activity, direct_message ),
+		.required = false,
+		.type = &json_field_bool
+	},
 
-	{ "attachment",	offsetof( struct ap_activity, attachments ),		false, &json_field_array_of, &json_field_string },
+	{ "attachment",	offsetof( struct ap_activity, attachments ),		false, &json_field_array_of, &ap_attachement_type },
 
 	JSON_FIELD_ARRAY_OF_STRINGS(to,true),
 	JSON_FIELD_ARRAY_OF_STRINGS(cc,false),
@@ -129,30 +122,13 @@ struct json_object_field ap_activity_layout[] = {
 
 	{ "signature",		offsetof( struct ap_activity, has_signature ),	false, &json_field_tagged_union, &signature_types },
 
-	{ "type",			offsetof( struct ap_activity, type ),			true,  &json_field_enum, ap_activity_type_enum },
+	JSON_FIELD_ENUM( type, ap_activity_type_enum, true ),
 
-	{ NULL, 			0,										true,  NULL,	&ap_activity_type },
+	JSON_FIELD_END,
 };
 #undef OBJ_TYPE
 
-
-static void ap_activity_free_shim( void* ptr )
-{
-	ap_activity_free(ptr);
-}
-static void* ap_activity_alloc()
-{
-	return (void*)ap_activity_new();
-}
-
-struct json_field_type ap_activity_type = {
-	.reader = json_field_object_type_reader,
-	.writer = json_field_object_type_writer,
-	.size = sizeof(struct ap_activity),
-	.layout = ap_activity_layout,
-	.alloc = ap_activity_alloc,
-	.free = ap_activity_free_shim,
-};
+JSON_FIELD_TYPE_OBJECT_LAYOUT_WITH_DEFAULTS( ap_activity );
 
 struct ap_activity* ap_activity_from_FILE( FILE* f )
 {

src/model/ap/activity/layout/content.c

@@ -0,0 +1,53 @@
+#include "../../activity.h"
+
+#include "json/json.h"
+
+#include <string.h>
+
+#define OBJ_TYPE struct ap_activity_source
+struct json_object_field ap_activity_source_layout[] = {
+	JSON_FIELD_STRING( content, true ),
+	JSON_FIELD_STRING( mime_type, false ),
+	JSON_FIELD_END,
+};
+#undef OBJ_TYPE
+
+bool ap_activity_source_reader( struct json_pull_parser* jpp, void* field_data, struct json_object_field* layout_field_data )
+{
+	struct ap_activity_source* src = field_data;
+
+	// If only a string is provided, that is the value field
+	char* data = json_pull_parser_read_string(jpp);
+	if( data ) {
+		memset(src,0,sizeof(*src));
+		src->content = data;
+		return true;
+	}
+
+	printf( "Reading suource as object\n" );
+	return json_read_object_layout( jpp, ap_activity_source_layout, src );
+}
+bool ap_activity_source_writer( struct json_writer* jw, const char* field_name, void* field_data, struct json_object_field* layout_field_data )
+{
+	struct ap_activity_source* src = field_data;
+
+	// Collapse to simple string if no additional properties are present
+	if( !src->mime_type ) {
+		json_write_field_name( jw, field_name );
+		if( !src->content ) {
+			json_write_string( jw->f, "" );
+		} else {
+			json_write_string( jw->f, src->content );
+		}
+		return true;
+	}
+
+	json_write_pretty_object_layout( jw, ap_activity_source_layout, src );
+	return true;
+}
+
+struct json_field_type ap_activity_source_type = {
+	.reader = ap_activity_source_reader,
+	.writer = ap_activity_source_writer,
+};
+

src/model/ap/activity/layout/signature.c

@@ -0,0 +1,21 @@
+#include "../../activity.h"
+
+static struct json_enum ap_signature_type_enum[] = {
+	{ "RsaSignature2017", 1 },
+	{ NULL, 0 },
+};
+
+#define OBJ_TYPE struct ap_signature
+struct json_object_field ap_signature_layout[] = {
+	JSON_FIELD_ENUM( type, ap_signature_type_enum, true ),
+	JSON_FIELD_STRING( creator, true ),
+	JSON_FIELD_DATETIME( created, true ),
+	{
+		.key = "signatureValue",
+		.offset = offsetof( OBJ_TYPE, value ),
+		.required = true,
+		.type = &json_field_string
+	},
+	JSON_FIELD_END,
+};
+#undef OBJ_TYPE

src/model/ap/activity/tag.c

@@ -28,17 +28,19 @@ void ap_activity_tag_free( struct ap_activity_tag* tag )
 	free(tag);
 }
 
-static void* alloc()
+static void* ap_activity_tag_alloc_shim()
 {
 	struct ap_activity_tag* ptr = malloc(sizeof(struct ap_activity_tag));
 	memset(ptr,0,sizeof(*ptr));
 	return ptr;
 }
-static void free_shim( void* ptr )
+static void ap_activity_tag_free_shim( void* ptr )
 {
 	ap_activity_tag_free(ptr);
 }
 
+JSON_FIELD_TYPE_OBJECT_LAYOUT( ap_activity_tag );
+/*
 struct json_field_type ap_activity_tag_type = {
 	.reader = json_field_object_type_reader,
 	.writer = json_field_object_type_writer,
@@ -47,4 +49,5 @@ struct json_field_type ap_activity_tag_type = {
 	.alloc = alloc,
 	.free = free_shim,
 };
+*/
 

src/model/status.c

@@ -86,7 +86,7 @@ struct status* status_from_uri( const char* uri )
 
 	return status_from_id(id);
 }
-static bool status_sync_from_activity_pub( struct status* s, struct ap_activity* act )
+bool status_sync_from_activity_pub( struct status* s, struct ap_activity* act )
 {
 	printf( "Syncing status from activity %s\n", act->id );
 	bool result = false;
@@ -99,9 +99,25 @@ static bool status_sync_from_activity_pub( struct status* s, struct ap_activity*
 	s->published = act->published;
 	s->remote = true;
 
-	s->source = strdup(act->source.value);
+	s->source = strdup(act->source.content);
 	s->url = strdup( act->id );
 
+	// Erase existing media
+	for( int i = 0; i < s->media.count; ++i ) {
+		free( s->media.items[i] );
+	}
+	free(s->media.items);
+	memset(&s->media,0,sizeof(s->media));
+
+	// Recreate the media field
+	for( int i = 0; i < act->attachments.count; ++i ) {
+		struct ap_attachement* att = act->attachments.items[i];
+		if( att && att->url ) {
+			char* media = strdup( att->url );
+			array_append( &s->media, sizeof(char*), &media );
+		}
+	}
+
 	status_save(s);
 	result = true;
 cleanup:
@@ -111,27 +127,14 @@ failed:
 	result = false;
 	goto cleanup;
 };
-struct status* status_fetch_from_uri( const char* uri )
+bool status_sync_from_uri( struct status* s, const char* uri )
 {
 	struct ap_activity* act = NULL;
 	FILE* f = NULL;
-	struct status* s = status_from_uri(uri);
-	if( !s ) {
-		s = malloc(sizeof(*s));
-		memset(s,0,sizeof(*s));
-
-		s->remote = true;
-		s->stub = true;
-		s->published = time(NULL);
-		s->url = strdup(uri);
-		status_save_new(s);
-
-		hash_index_set( "data/statuses/uri", uri, s->id );
-	}
 
 	// Fetch the object from the remote server
 	char filename[512];
-	snprintf( filename, sizeof(filename), "data/statuses/%d.ap.json", s->id );
+	snprintf( filename, sizeof(filename), "data/statuses/ap/%d.json", s->id );
 	f = fopen(filename,"r");
 	if( !f ) {
 		printf( "* Fetching %s\n", uri );
@@ -183,6 +186,33 @@ failed:
 	s = NULL;
 	goto cleanup;
 }
+bool status_sync( struct status* s )
+{
+	return status_sync_from_uri( s, s->url );
+}
+struct status* status_fetch_from_uri( const char* uri )
+{
+	struct status* s = status_from_uri(uri);
+	if( !s ) {
+		s = malloc(sizeof(*s));
+		memset(s,0,sizeof(*s));
+
+		s->remote = true;
+		s->stub = true;
+		s->published = time(NULL);
+		s->url = strdup(uri);
+		status_save_new(s);
+
+		hash_index_set( "data/statuses/uri", uri, s->id );
+	}
+
+	if( !status_sync_from_uri(s,uri) ) {
+		status_free(s);
+		return NULL;
+	}
+
+	return s;
+}
 struct status* status_new_system_unfollow( int account_id )
 {
 	struct account* a = account_from_id(account_id);
@@ -316,9 +346,16 @@ void status_flag_for_async_fetch( struct status* s )
 
 	json_read_object_layout_from_file( "data/statuses/async_fetch.json", async_fetch_layout, &fetch );
 
+	// Don't add if already flagged for async fetch
+	for( int i = 0; i < fetch.ids.count; ++i ) {
+		if( fetch.ids.items[i] == s->id ) {
+			goto already_present;
+		}
+	}
 	array_append( &fetch.ids, sizeof(s->id), &s->id );
 
 	json_write_object_layout_to_file( "data/statuses/async_fetch.json", "\t", async_fetch_layout, &fetch );
+already_present:
 	free( fetch.ids.items );
 }
 

src/model/status.h

@@ -59,6 +59,9 @@ struct status* status_fetch_from_uri( const char* uri );
 
 struct ap_activity;
 struct status* status_from_activity( struct ap_activity* act );
+bool status_sync_from_activity_pub( struct status* s, struct ap_activity* act );
+bool status_sync_from_uri( struct status* s, const char* uri );
+bool status_sync( struct status* s );
 
 bool status_save_new( struct status* s );
 void status_write_to_FILE( struct status* s, FILE* f );