teknomunk
/
apogee
1
0
Fork 0
Code Issues 17 Pull Requests Packages Projects Releases Wiki Activity

Add code to pull followers/following list, rework fetching, rework account sync

Browse Source
master
teknomunk 1 year ago
parent 3c2f038ecc
commit 22bae76274
13 changed files with 328 additions and 124 deletions
Show Stats Download Patch File Download Diff File

2
src/ap

@ -1 +1 @@
Subproject commit c1a1148184cd335366b97a2efaae116a526b6de9
Subproject commit 1ff885b9b810503a2d98b8f78fb749ed07cd7ccf

2
src/collections

@ -1 +1 @@
Subproject commit af6ffed1c98d57b754692ab8f55467a5215a3bd2
Subproject commit 44a552b831da2c1d9556fe909b6c77614f99a59e

18
src/controller/cli.c
Unescape View File

@ -192,6 +192,23 @@ bool handle_command_account_sync( struct cli_request* req, int account_id )
return true;
}
bool handle_command_account_pull( struct cli_request* req, int account_id )
{
int res = cli_route_command( req, "pull", 0, "" );
if( res != 1 ) { return !!res; }
printf( "Pulling account followers/following lists %d\n", account_id );
struct account* a = account_from_id(account_id);
if( !a ) {
printf( "Account %d does not exist\n", account_id );
return true;
}
account_sync_following_list(a);
account_sync_followers_list(a);
account_save(a);
return true;
}
bool handle_command_update( struct cli_request* req )
{
int res = cli_route_command( req, "update", 0, "update" );
@ -215,6 +232,7 @@ bool handle_command_account( struct cli_request* req )
req->argc -= 1;
if( handle_command_account_sync(req,account_id) ) { goto done; }
if( handle_command_account_pull(req,account_id) ) { goto done; }
result = false;
cleanup:

2
src/controller/inbox/add.c
Unescape View File

@ -23,7 +23,7 @@ bool route_add( struct ap_object* act )
printf( "Don't have account...\n" );
goto failed;
} // Don't have ActivityPub account data, try again later
apa = account_activity_pub(a);
apa = account_get_activity_pub_data(a);
if( 0 != strcmp( apa->featured, act->target ) ) {
printf( "Not adding to featured...\n" );
printf( "apa->featured = %s\n", apa->featured );

3
src/controller/owner.c
Unescape View File

@ -72,6 +72,7 @@ failed:
}
static bool handle_following( struct http_request* req )
{
printf( "handle_following\n" );
bool result = false;
struct account* owner_account = account_from_id(0);
@ -158,7 +159,7 @@ bool route_owner( struct http_request* req )
if( http_request_route_term( req, "" ) ) {
return handle_owner_actor(req);
}
} else if( http_request_route_term( req, "/following" ) ) {
} else if( http_request_route( req, "/following" ) ) {
return handle_following(req);
} else if( http_request_route( req, "/followers" ) ) {
return handle_followers(req);

2
src/json

@ -1 +1 @@
Subproject commit 934dcd3d7f92975680b7c1a4574a86010417c7c7
Subproject commit 1d59e38974df4022d3d84af7ae8e765b76611461

66
src/model/account.c
Unescape View File

@ -117,11 +117,17 @@ static struct json_object_field account_layout[] = {
JSON_FIELD_END,
};
static struct account* new_system_account()
struct account* account_new()
{
struct account* a;
a = malloc(sizeof(*a));
memset(a,0,sizeof(*a));
return a;
}
static struct account* new_system_account()
{
struct account* a = account_new();
a->id = system_account_id;
a->handle = strdup("system");
@ -164,8 +170,7 @@ struct account* account_from_id( int id )
char filename[512];
snprintf( filename, 512, "data/accounts/%d.json", id );
struct account* a = malloc(sizeof(struct account));
memset( a, 0, sizeof(struct account) );
struct account* a = account_new();
a->id = id;
if( !json_read_object_layout_from_file( filename, account_layout, a ) ) {
@ -188,7 +193,8 @@ static bool index_uri_to_account_id( const char* uri, int account_id )
{
return hash_index_set( "data/accounts/uri_index/", uri, account_id );
}
static int lookup_account_id_from_uri( const char* uri )
int account_id_from_uri( const char* uri )
{
int result = 0;
if( !hash_index_get( "data/accounts/uri_index/", uri, &result ) ) {
@ -197,6 +203,10 @@ static int lookup_account_id_from_uri( const char* uri )
return result;
}
static int lookup_account_id_from_uri( const char* uri )
{
return account_id_from_uri(uri);
}
struct account* account_from_uri( const char* uri )
{
@ -213,7 +223,7 @@ struct account* account_from_uri( const char* uri )
int account_id = lookup_account_id_from_uri( uri );
if( account_id == -1 ) {
printf( "Failed to lookup local account id for %s\n", uri );
//printf( "Failed to lookup local account id for %s\n", uri );
return NULL;
}
@ -262,7 +272,7 @@ struct crypto_keys* account_get_public_key( struct account* a, const char* key_n
char filename[512];
FILE* f = fopen( format( filename, sizeof(filename), "data/accounts/%d/%s.pem", a->id, key_name ), "r" );
if( !f ) {
printf( "Failed to open file %s\n", filename );
//printf( "Failed to open file %s\n", filename );
return NULL;
}
fclose(f);
@ -308,27 +318,13 @@ struct account* account_fetch_from_uri( const char* uri )
struct account* a = NULL;
a = account_from_id(account_id);
// Fetch the ActivityPub actor data if we don't already have it
char filename[512];
snprintf( filename,sizeof(filename), "data/accounts/%d/ap.json", account_id );
struct stat s;
bool fetch_attempted = false;
if( !stat( filename, &s ) || ( time(NULL) - s.st_mtime > 60*60*24*3 ) ) {
pull_remote_file( filename, uri );
fetch_attempted = true;
}
FILE* f = fopen( filename, "r" );
if( !f ) {
if( a && a->stub && ( time(NULL) < a->next_stub_recheck ) ) { goto next; }
if( !fetch_attempted && pull_remote_file( filename, uri ) ) { goto next; }
if( a ) {
a->id = account_id;
a->account_url = strdup(uri);
account_save(a);
} else {
// Mark as stub
if( !a ) {
a = malloc(sizeof(*a));
memset(a,0,sizeof(*a));
}
if( !a ) { a = account_new(); }
a->id = account_id;
a->account_url = strdup(uri);
@ -338,16 +334,26 @@ struct account* account_fetch_from_uri( const char* uri )
account_save(a);
account_index_webfinger(a);
account_free(a);
}
// Fetch the ActivityPub actor data if we don't already have it
char filename[512];
snprintf( filename,sizeof(filename), "data/accounts/%d/ap.json", account_id );
pull_remote_file_if_older( filename, uri, 60*60*24*3 );
FILE* f = fopen( filename, "r" );
if( !f ) {
return NULL;
next:
/* nop */;
} else {
printf( "! AP data is present at %s\n", filename );
fclose(f);
}
account_free(a);
// Fail if we can't sync
if( !account_sync_from_activity_pub( account_id ) ) {
printf( "Failed to sync from activity pub data (account_id=%d)\n", account_id );
return NULL;
}
@ -424,7 +430,7 @@ void account_add_follower( struct account* a, struct account* follower )
account_save(follower);
// Create notification for follow
if( follower->id != owner_account_id ) {
if( ( a->id == owner_account_id ) && ( follower->id != owner_account_id ) ) {
struct notification* note = notification_new();
note->debug = 4;
note->type = nt_follow;

5
src/model/account.h
Unescape View File

@ -92,6 +92,7 @@ void account_index_webfinger( struct account* a );
struct account* account_from_id( int id );
struct account* account_from_uri( const char* uri );
int account_id_from_uri( const char* uri );
struct account* account_from_webfinger( const char* handle );
struct account* account_new();
struct account* account_from_uri_or_fetch( const char* uri );
@ -102,7 +103,9 @@ void account_save( struct account* a );
// Update from external activity pub data
bool account_sync_from_activity_pub( unsigned int id );
bool account_sync_from_activity( struct account* a, struct ap_object* act );
struct ap_object* account_get_activity_pub_data( struct account* a );
void account_sync_following_list( struct account* a );
void account_sync_followers_list( struct account* a );
// Data requests
struct crypto_keys* account_get_public_key( struct account* a, const char* key_name );
@ -111,7 +114,7 @@ void account_list_followers( struct account* a, int offset, int limit, void* id_
void account_list_following( struct account* a, int offset, int limit, void* id_array );
// Activity pub data
struct ap_object* account_activity_pub( struct account* a );
//struct ap_object* account_activity_pub( struct account* a );
struct ap_object* account_ap_actor( struct account* a );
struct ap_object* account_ap_outbox( struct account* a );
struct ap_object* account_ap_outbox_page( struct account* a, int page );

16
src/model/account/ap_data.c
Unescape View File

@ -15,6 +15,7 @@
#include <stdlib.h>
#include <string.h>
/*
struct ap_object* account_activity_pub( struct account* a )
{
// Pass thru remote activity pub actor data if present
@ -24,6 +25,7 @@ struct ap_object* account_activity_pub( struct account* a )
return account_ap_actor(a);
}
*/
struct ap_object* account_ap_actor( struct account* a )
{
@ -175,15 +177,15 @@ struct ap_object* account_ap_outbox_page( struct account* a, int page )
outbox->prev = aformat( "https://%s/outbox/page-%d", g_server->domain, page - 1 );
}
if( page < page_count - 1 ) {
outbox->next = aformat( "https://%s/outbox/page-%d", g_server->domain, page + 1 );
outbox->next.tag = apaot_ref;
outbox->next.ref = aformat( "https://%s/outbox/page-%d", g_server->domain, page + 1 );
}
return outbox;
}
static struct ap_object* account_list_page( int page, char* part_of, const char* page_format, const char* trie_filename )
{
enum { items_per_page = 10 };
char buffer[512];
enum { items_per_page = 50 };
int total_items = ffdb_trie_count( trie_filename );
int page_count = ( total_items + items_per_page - 1 ) / items_per_page;
@ -196,8 +198,9 @@ static struct ap_object* account_list_page( int page, char* part_of, const char*
int count;
} values;
memset( &values, 0, sizeof(values) );
ffdb_trie_list( buffer, page * items_per_page, items_per_page, &values, NULL );
ffdb_trie_list( trie_filename, page * items_per_page, items_per_page, &values, NULL );
for( int i = 0; i < values.count; ++i ) {
printf( "values[%d] = %s\n", i, values.items[i] );
int id;
if( 1 != sscanf( values.items[i], "%d", &id ) ) { continue; }
free( values.items[i] );
@ -208,7 +211,7 @@ static struct ap_object* account_list_page( int page, char* part_of, const char*
struct ap_object_ptr_or_ref r;
r.tag = apaot_ref;
r.ref = strdup( follower->account_url );
array_append( &o->collection_items, sizeof(r), &r );
array_append( &o->ordered_items, sizeof(r), &r );
account_free(follower);
}
free( values.items );
@ -221,7 +224,8 @@ static struct ap_object* account_list_page( int page, char* part_of, const char*
o->prev = aformat( page_format, page - 1 );
}
if( page < page_count - 1 ) {
o->next = aformat( page_format, page + 1 );
o->next.tag = apaot_ref;
o->next.ref = aformat( page_format, page + 1 );
}
return o;

223
src/model/account/ap_sync.c
Unescape View File

@ -7,52 +7,84 @@
#include "collections/array.h"
#include "util/format.h"
#include "ap/object.h"
#include "ap/collection.h"
// Standard Library
#include <string.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>
bool account_sync_from_activity_pub( unsigned int account_id )
struct ap_object* account_get_activity_pub_data( struct account* a )
{
bool result = false;
struct account* a = NULL;
if( !a ) { return NULL; }
if( !a->account_url ) {
printf( "No account url\n" );
goto skip_pull;
}
char filename[512];
snprintf( filename, 512, "data/accounts/%d/ap.json", account_id );
snprintf( filename, 512, "data/accounts/%d/ap.json", a->id );
printf( "ap_object_from_file( %s )\n", filename );
struct ap_object* obj = ap_object_from_file(filename);
pull_remote_file_if_older( filename, a->account_url, 60*60*24*3 );
skip_pull:
return ap_object_from_file( filename );
}
bool account_sync_from_activity_pub( unsigned int account_id )
{
bool result = false;
struct account* a = account_from_id(account_id);
if( !a ) {
a = account_new();
a->id = account_id;
}
struct ap_object* obj = account_get_activity_pub_data(a);
if( !obj ) {
// Try to fetch using the account_url field
a = account_from_id(account_id);
if( !a ) { goto failed; }
pull_remote_file( filename, a->account_url );
obj = ap_object_from_file(filename);
account_free(a); a = NULL;
if( !obj ) { goto failed; }
} else {
struct stat s;
stat( filename, &s );
if( time(NULL) - s.st_mtime > 60*60*24*3 ) {
// Refresh if over 3 days old
pull_remote_file( filename, obj->id );
ap_object_free(obj);
obj = ap_object_from_file(filename);
if( !obj ) { goto failed; }
}
printf( "Failed to load AP data\n" );
goto failed;
}
printf( "obj = "); ap_object_write_to_FILE(obj,stdout);
printf( "\n" );
if( !account_sync_from_activity( a, obj ) ) { goto failed; }
a = malloc(sizeof(struct account));
memset(a,0,sizeof(*a));
a->id = account_id;
account_save(a);
account_index_webfinger(a);
if( !obj ) {
printf( "? Failed to sync account %d from %s, creating stub\n", account_id, filename );
goto failed;
goto succeeded;
succeeded:
result = true;
goto cleanup;
cleanup:
ap_object_free(obj);
account_free(a);
return result;
failed:
result = false;
goto cleanup;
}
bool account_sync_from_activity( struct account* a, struct ap_object* obj )
{
if( obj->url ) {
a->account_url = strdup(obj->url);
} else if( obj->id ) {
a->account_url = strdup(obj->id);
} else {
printf( "No id or url in AP data\n" );
return false;
}
for( int i = 0; i < a->aliases.count; ++i ) {
free( a->aliases.items[i] );
}
a->aliases.count = 0;
for( int i = 0; i < obj->also_known_as.count; ++i ) {
char* str = strdup(obj->also_known_as.items[i]);
array_append( &a->aliases, sizeof(str), &str );
}
a->handle = strdup(obj->preferred_username);
@ -70,13 +102,7 @@ bool account_sync_from_activity_pub( unsigned int account_id )
}
a->bot = ( obj->type != ap_Person );
a->account_type = at_remote_activity_pub;
if( obj->url ) {
a->account_url = strdup(obj->url);
} else if( obj->id ) {
a->account_url = strdup(obj->id);
} else {
goto failed;
}
if( obj->inbox ) {
a->inbox = strdup(obj->inbox);
}
@ -116,6 +142,7 @@ bool account_sync_from_activity_pub( unsigned int account_id )
}
// Extract out the public key
char filename[512];
char* id = strdup(obj->public_key->id);
char* key_id = NULL;
strtok_r( id, "#", &key_id );
@ -129,40 +156,112 @@ bool account_sync_from_activity_pub( unsigned int account_id )
}
free(id);
account_save(a);
account_index_webfinger(a);
goto succeeded;
succeeded:
result = true;
goto cleanup;
cleanup:
ap_object_free(obj);
account_free(a);
return true;
}
return result;
failed:
result = false;
goto cleanup;
static struct ap_object* fetch_ap_object_with_delay( const char* uri )
{
// Sleep for 150 milliseconds to reduce load on remote servers
usleep( 150 * 1000 );
return fetch_ap_object_ref(uri);
}
bool account_sync_from_activity( struct account* a, struct ap_object* act )
void account_sync_following_list( struct account* a )
{
for( int i = 0; i < a->aliases.count; ++i ) {
free( a->aliases.items[i] );
}
a->aliases.count = 0;
ap_object_set_fetch_callback( fetch_ap_object_with_delay );
for( int i = 0; i < act->also_known_as.count; ++i ) {
char* str = strdup(act->also_known_as.items[i]);
array_append( &a->aliases, sizeof(str), &str );
struct ap_object* obj = NULL;
struct ap_object* following = NULL;
if( a->local ) { goto cleanup; }
obj = account_get_activity_pub_data(a);
if( !obj ) { goto cleanup; }
following = ap_collection_from_uri( obj->following );
if( !following ) { goto cleanup; }
struct {
int* items;
int count;
} existing;
account_list_following( a, 0, INT_MAX, &existing );
for( struct ap_object* page = following->first.ptr; page; ap_collection_iterate(&page) ) {
for( int i = 0; i < page->ordered_items.count; ++i ) {
// ap_object_force_to_ref( &page->ordered_items.items[i] );
const char* account_uri = page->ordered_items.items[i].ref;
struct account* following = account_from_uri_or_fetch( account_uri );
if( !following ) { continue; }
int account_id = following->id;
printf( "Follower local account at %d\n", account_id );
if( -1 != array_find( &existing, sizeof(account_id), &account_id ) ) {
array_delete( &existing, sizeof(account_id), &account_id );
} else {
printf( "Adding following %s (%d)\n", account_uri, account_id );
account_add_follower( following, a );
}
account_free(following);a
}
}
return true;
printf( "existing.count = %d\n", existing.count );
for( int i = 0; i < existing.count; ++i ) {
printf( "Need to remove %d as following\n", existing.items[i] );
}
free(existing.items);
cleanup:
ap_object_free(obj);
ap_object_free(following);
}
void account_sync_following_list( struct account* a )
void account_sync_followers_list( struct account* a )
{
// TODO: implement
ap_object_set_fetch_callback( fetch_ap_object_with_delay );
struct {
int* items;
int count;
} existing;
account_list_followers( a, 0, INT_MAX, &existing );
struct ap_object* obj = account_get_activity_pub_data(a);
if( !obj->followers ) { goto cleanup; }
struct ap_object* followers = ap_collection_from_uri( obj->followers );
if( !followers ) { goto cleanup; }
for( struct ap_object* page = followers->first.ptr; page; ap_collection_iterate(&page) ) {
for( int i = 0; i < page->ordered_items.count; ++i ) {
// ap_object_force_to_ref( &page->ordered_items.items[i] );
const char* account_uri = page->ordered_items.items[i].ref;
struct account* follower = account_from_uri_or_fetch( account_uri );
if( !follower ) { continue; }
int account_id = follower->id;
printf( "Follower local account at %d\n", follower->id );
if( -1 != array_find( &existing, sizeof(account_id), &account_id ) ) {
array_delete( &existing, sizeof(account_id), &account_id );
} else {
printf( "Adding follower %s (%d)\n", account_uri, account_id );
account_add_follower( a, follower );
}
account_free(follower);
}
}
printf( "existing.count = %d\n", existing.count );
for( int i = 0; i < existing.count; ++i ) {
printf( "Need to remove %d as follower\n", existing.items[i] );
}
free(existing.items);
cleanup:
ap_object_free(followers);
}

2
src/model/account/fed_act.c
Unescape View File

@ -220,7 +220,7 @@ void account_update( struct account* a )
act->actor = strdup(a->account_url);
act->object.tag = apaot_activity;
act->object.ptr = account_activity_pub(a);
act->object.ptr = account_get_activity_pub_data(a);
char* str = aformat("https://%s/owner/followers", g_server->domain );
array_append( &act->to, sizeof(str), &str );

104
src/model/fetch.c
Unescape View File

@ -1,48 +1,114 @@
#define _GNU_SOURCE
#include "fetch.h"
// Submodules
#include "http/client/client.h"
#include "util/format.h"
#include "ap/object.h"
#include "sha256/sha256.h"
// Standard Library
#include <sys/stat.h>
#include <time.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
bool pull_remote_file( const char* filename, const char* uri )
static bool do_fetch( const char* uri, FILE* result )
{
printf( "* Fetching %s\n", uri );
char tmp_filename[512];
FILE* f = fopen(format(tmp_filename,512,"%s.tmp",filename),"w");
long status_code = -1;
long status_code;
const void* request[] = {
HTTP_REQ_URL, uri,
HTTP_REQ_HEADER, "Accept: application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"",
HTTP_REQ_OUTFILE, f,
HTTP_REQ_OUTFILE, result,
HTTP_REQ_RESULT_STATUS, &status_code,
NULL,
};
printf( "GET %s\n", uri );
if( !http_client_do( request ) ) {
printf( "GET %s -> %ld\n", uri, status_code );
printf( "! Unable to fetch %s\n", uri );
fclose(f);
return false;
}
printf( "GET %s -> %ld\n", uri, status_code );
printf( "status_code = %ld\n", status_code );
if( status_code == 200 ) {
// success
fclose(f);
rename(tmp_filename,filename);
return true;
} else if( status_code == 401 ) {
// Not Authorized
if( status_code == 401 ) {
// TODO: do signed fetch
printf( "TODO: perform signed fetch\n" );
fclose(f);
return false;
}
return ( status_code == 200 );
}
// Failure
bool pull_remote_file( const char* filename, const char* uri )
{
printf( "* Fetching %s\n", uri );
char tmp_filename[512];
FILE* f = fopen(format(tmp_filename,512,"%s.tmp",filename),"w");
bool result = false;
if( do_fetch( uri, f ) ) {
rename(tmp_filename,filename);
result = true;
}
fclose(f);
return false;
return result;
}
bool pull_remote_file_if_older( const char* filename, const char* uri, int seconds )
{
struct stat s;
char tmp_filename[512];
snprintf( tmp_filename,512, "%s.tmp", filename );
// Skip download if .tmp file exists from a failed fetch and is within the timeout window
if( 0 == stat(tmp_filename, &s ) ) {
if( time(NULL) - s.st_mtime <= seconds ) {
return false;
}
}
if( 0 != stat(filename, &s ) ) { goto pull; }
if( time(NULL) - s.st_mtime > seconds ) { goto pull; }
return true;
pull:
return pull_remote_file( filename, uri );
}
char* fetch_remote_file_to_cache( const char* uri, int seconds )
{
char* filename = NULL;
char hash[65];
sha256_easy_hash_hex( uri, strlen(uri), hash );
hash[64] = 0;
mkdir( "data/cache", 0755 );
asprintf( &filename, "data/cache/%s.dat", hash );
printf( "Using %s for %s\n", filename, uri );
if( !pull_remote_file_if_older(filename,uri,seconds) ) {
free(filename);
return NULL;
}
return filename;
}
enum {
LIFE = 60*60*24*2 // 2 days
};
struct ap_object* fetch_ap_object_ref( const char* uri )
{
char* filename = fetch_remote_file_to_cache(uri, LIFE );
if( !filename ) { return NULL; }
struct ap_object* res = ap_object_from_file(filename);
free(filename);
return res;
}

7
src/model/fetch.h
Unescape View File

@ -2,5 +2,12 @@
#include <stdbool.h>
struct ap_object_ptr_or_ref;
bool pull_remote_file( const char* filename, const char* uri );
bool pull_remote_file_if_older( const char* filename, const char* uri, int seconds );
char* fetch_remote_file_to_cache( const char* uri, int seconds );
struct ap_object* fetch_ap_object_ref( const char* uri );

Write Preview
Loading…
Cancel
Save