Handle indepotency key, discard posts with content or source larger than 4000 bytes, start work in signed fetch (not working yet), update submodules to fix inbox processing bug

src/ap

@@ -1 +1 @@
-Subproject commit 3996ded88621cc14518bbb964aa65aa99e7ef110
+Subproject commit 085331a08384befe0b68f217240a305c05040377

src/controller/api/status.c

@@ -22,6 +22,9 @@
 // Controller
 #include "controller/api/client_apps.h"
 
+// Submodules
+#include "ffdb/trie.h"
+
 // Standard Library
 #include <stdio.h>
 #include <stdlib.h>
@@ -114,6 +117,16 @@ bool handle_post( struct http_request* req, struct account* a )
 	} params;
 	memset(&params,0,sizeof(params));
 
+	const char* indempotency_key = http_request_get_header( req, "Idempotency-Key" );
+	if( indempotency_key ) {
+		char* existing = ffdb_trie_get( "data/indempotency", indempotency_key );
+		if( existing ) {
+			s = status_from_id( atoi(existing) );
+			free( existing );
+			goto success;
+		}
+	}
+
 	#define OBJ_TYPE struct params_t
 	static struct json_object_field layout[] = {
 		JSON_FIELD_ARRAY_OF_STRINGS( media_ids, false ),
@@ -137,6 +150,10 @@ bool handle_post( struct http_request* req, struct account* a )
 	s->source = strdup( params.status );
 	status_save_new(s);
 
+	char key[32];
+	snprintf( key,32, "%d", s->id );
+	ffdb_trie_set( "data/indempotency", indempotency_key, key );
+
 	for( int i = 0; i < params.media_ids.count; ++i ) {
 		struct media* m = media_from_id( atoi(params.media_ids.items[i]) );
 		if( !m ) { continue; }
@@ -166,6 +183,7 @@ bool handle_post( struct http_request* req, struct account* a )
 	http_request_send_headers( req, 200, "application/json", true );
 	FILE* f = http_request_get_response_body(req);
 	api_Status_write( s, f, 0 );
+success:
 	result = true;
 cleanup:
 	free(params.status);

src/controller/inbox.c

@@ -385,6 +385,12 @@ static bool route_create( struct ap_object* act )
 		goto failed;
 	}
 	struct ap_object* obj = act->object.ptr;
+	if( obj->content.content ) {
+		if( strlen( obj->content.content ) > 4000 ) { goto discard; }
+	}
+	if( obj->source.content ) {
+		if( strlen( obj->source.content ) > 4000 ) { goto discard; }
+	}
 
 	// Does this activity have mention me
 	char owner_url[512];
@@ -537,7 +543,10 @@ static bool process_one()
 	// Load activity
 	FILE* f = fmemopen( env->body, strlen(env->body), "r" );
 	act = ap_object_from_FILE(f);
-	if( !act ) { goto failed; }
+	if( !act ) {
+		printf( "Failed to parse body: %s\n", env->body );
+		goto failed;
+	}
 
 	if( handle_forward( env, act ) ) { goto discard; }
 
@@ -558,8 +567,6 @@ static bool process_one()
 	// Validate signature
 	env->validated = http_signature_validate( env, "post /inbox", act->actor );
 
-
-
 	if( !env->validated ) { goto discard; }
 
 	printf( "Processing %d\n", id );

src/controller/outbox.c

@@ -77,7 +77,11 @@ static bool process_envelope( struct outbox_envelope* env )
 	}
 
 	struct http_signature hs;
-	if( !http_signature_make( inbox, keys, &hs, postdata ) ) {
+	memset( &hs, 0, sizeof(hs) );
+	hs.input.url = inbox;
+	hs.input.method = "post";
+	hs.input.post_data = postdata;
+	if( !http_signature_make( keys, &hs ) ) {
 		printf( "! Failed to make HTTP signature\n" );
 		goto failed;
 	}

src/controller/test/crypto.c

@@ -83,7 +83,11 @@ static bool test_http_signature_2()
 	keys = crypto_keys_new();
 	crypto_keys_load_private( keys, "assets/test.private.pem" );
 
-	if( !http_signature_make( "https://example.com/inbox", keys, &hs, "{\"hello\": \"world\"}" ) ) { goto failed; }
+	memset(&hs,0,sizeof(hs));
+	hs.input.method = "post";
+	hs.input.url = "https://example.com/inbox";
+	hs.input.post_data = "{\"hello\": \"world\"}";
+	if( !http_signature_make( keys, &hs ) ) { goto failed; }
 
 	char signature_header[512];
 	snprintf( signature_header, sizeof(signature_header), "keyId=\"Test\",headers=\"(request-target) host date content-length digest\",signature=\"%s\"", hs.signature );

src/json

@@ -1 +1 @@
-Subproject commit d5ab61ee7297302aff9391e8809f94a949a03f3c
+Subproject commit c8d14e2ba30c3536911c31f5db935a269f179ba0

src/model/crypto/http_sign.c

@@ -15,8 +15,13 @@
 #include <string.h>
 #include <time.h>
 
-bool http_signature_make( const char* inbox, struct crypto_keys* keys, struct http_signature* sign, const char* postdata )
+bool http_signature_make( struct crypto_keys* keys, struct http_signature* sign )
 {
+	const char* inbox = sign->input.url;
+	const char* method = sign->input.method;
+	if( !method ) { method = "post"; }
+	const char* postdata = sign->input.post_data;
+
 	memset(sign,0,sizeof(*sign));
 
 	if( 0 != strncmp( "https://", inbox, 8 ) ) {
@@ -49,7 +54,11 @@ bool http_signature_make( const char* inbox, struct crypto_keys* keys, struct ht
 
 	// Calculate Digest
 	unsigned char raw_hash[32];
-	sign->content_length = strlen(postdata);
+	if( postdata ) {
+		sign->content_length = strlen(postdata);
+	} else {
+		sign->content_length = 0;
+	}
 	sha256_easy_hash( postdata, sign->content_length, raw_hash );
 	char* base64_hash = base64_strict_encode(raw_hash,32);
 	sign->digest = aformat( "SHA-256=%s", base64_hash );
@@ -58,8 +67,8 @@ bool http_signature_make( const char* inbox, struct crypto_keys* keys, struct ht
 	// Build hash line
 	char hash_line[512];
 	snprintf( hash_line, 512,
-		"(request-target): post %s\nhost: %s\ndate: %s\ncontent-length: %d\ndigest: %s",
-		path, host, date, sign->content_length, sign->digest
+		"(request-target): %s %s\nhost: %s\ndate: %s\ncontent-length: %d\ndigest: %s",
+		method, path, host, date, sign->content_length, sign->digest
 	);
 	printf( "\nbuilding hash_line = %s|\n\n", hash_line );
 

src/model/crypto/http_sign.h

@@ -7,6 +7,13 @@ struct crypto_keys;
 struct ap_envelope;
 struct http_signature
 {
+	struct
+	{
+		const char* method;
+		const char* url;
+		const char* post_data;
+	} input;
+
 	char* host;
 	char* date;
 	char* signature;
@@ -14,7 +21,7 @@ struct http_signature
 	int content_length;
 };
 
-bool http_signature_make( const char* inbox, struct crypto_keys* keys, struct http_signature* sign, const char* postdata );
+bool http_signature_make( struct crypto_keys* keys, struct http_signature* sign );
 void http_signature_free( struct http_signature* sign );
 bool http_signature_validate( struct ap_envelope* env, const char* request_target, const char* expected_actor );
 

src/model/fetch.c

@@ -1,6 +1,11 @@
 #define _GNU_SOURCE
 #include "fetch.h"
 
+// Model
+#include "model/crypto/keys.h"
+#include "model/crypto/http_sign.h"
+#include "model/server.h"
+
 // Submodules
 #include "http/client/client.h"
 #include "util/format.h"
@@ -16,27 +21,76 @@
 
 static bool do_fetch( const char* uri, FILE* result )
 {
+	char user_agent[512];
+	snprintf( user_agent, sizeof(user_agent), "User-Agent: curl (Apogee/0.1; +https://%s/owner/actor)", g_server->domain );
+
 	long status_code;
 	const void* request[] = {
 		HTTP_REQ_URL, uri,
 		HTTP_REQ_HEADER, "Accept: application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"",
+		HTTP_REQ_HEADER, user_agent,
 		HTTP_REQ_OUTFILE, result,
 		HTTP_REQ_RESULT_STATUS, &status_code,
 		NULL,
 	};
 	printf( "GET %s\n", uri );
-	if( !http_client_do( request ) ) {
-		printf( "GET %s -> %ld\n", uri, status_code );
-		printf( "! Unable to fetch %s\n", uri );
-		return false;
-	}
+	http_client_do( request );
 	printf( "GET %s -> %ld\n", uri, status_code );
 
 	if( status_code == 401 ) {
+		printf( "Retrying request with HTTP Signature header...\n" );
+
+		// Load crypto keys
+		struct crypto_keys* keys = crypto_keys_new();
+		if( !crypto_keys_load_private( keys, "data/owner/private.pem" ) ) {
+			printf( "Failed to load private key\n" );
+			return false;
+		}
+
 		// TODO: do signed fetch
-		printf( "TODO: perform signed fetch\n" );
-		return false;
+		struct http_signature hs;
+		memset( &hs, 0, sizeof(hs) );
+		hs.input.method = "get";
+		hs.input.url = uri;
+
+		if( !http_signature_make( keys, &hs ) ) {
+			return false;
+		}
+
+		char date_header[512];
+		snprintf( date_header, sizeof(date_header), "Date: %s", hs.date );
+
+		char sign_header[512];
+		snprintf( sign_header, sizeof(sign_header), "Signature: keyId=\"https://%s/owner/actor#mainKey\",headers=\"(request-target) host date content-length digest\",signature=\"%s\"",
+			g_server->domain,
+			hs.signature
+		);
+
+		char digest_header[512];
+		snprintf( digest_header, sizeof(digest_header), "Digest: %s", hs.digest );
+
+		char content_length_header[512];
+		snprintf( content_length_header, sizeof(content_length_header), "Content-Length: %d", hs.content_length );
+
+		const void* request2[] = {
+			HTTP_REQ_URL, uri,
+			HTTP_REQ_HEADER, user_agent,
+			HTTP_REQ_HEADER, date_header,
+			HTTP_REQ_HEADER, sign_header,
+			HTTP_REQ_HEADER, content_length_header,
+			HTTP_REQ_HEADER, digest_header,
+			HTTP_REQ_HEADER, "Accept: application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"",
+			HTTP_REQ_OUTFILE, result,
+			HTTP_REQ_RESULT_STATUS, &status_code,
+			NULL,
+		};
+
+		if( !http_client_do( request2 ) ) {
+			printf( "Failed to perform get\n" );
+		}
+		printf( "GET %s -> %ld\n", uri, status_code );
 	}
+
 	return ( status_code == 200 );
 }