teknomunk
/
apogee
1
0
Fork 0
Code Issues 17 Pull Requests Packages Projects Releases Wiki Activity

Add build file (was excluded by .gitignore), update .gitignore, add automatic account sync (3-5 day interval), fix inconsistent actor/attributedTo data, move rfc3339 timestamp code to util submodule

Browse Source
master
teknomunk 10 months ago
parent a1bf8a8be8
commit 5b04add122
11 changed files with 159 additions and 36 deletions
Show Stats Download Patch File Download Diff File

3
.gitignore vendored
Unescape View File

@ -9,6 +9,7 @@ data/
assets/soapbox*
backup/
vgcore.*
apogee*
apogee
apogee.debug
/*.log
/*.json

2
src/ap

@ -1 +1 @@
Subproject commit 64797a74e8955aa500deaff1e6883b7349022892
Subproject commit 67471e0105e63d7fb407c89af0f0dbecf12695c2

35
src/apogee.build.json
Unescape View File

@ -0,0 +1,35 @@
{
"cxx":{
"cc": "gcc",
"ld": "gcc",
"ar": "ar",
"flags": "-g -MP -MD -Os -Og -Werror -flto -Wall -I./ -Isrc/",
"ldflags": "-lcurl -lssl -lcrypto -flto -Os",
"output": "obj/apogee",
"objdir": "obj/apogee/"
},
"tools": [ "cxx" ],
"sources": [
"src/ap",
"src/collections",
"src/controller",
"src/ffdb",
"src/http",
"src/json",
"src/json-ld",
"src/model.c",
"src/model",
"src/rdf",
"src/reflect",
"src/rsa-signature-2017",
"src/sha256/sha256.c",
"src/util",
"src/view",
"src/dev.c",
"src/form.c",
"src/tor.c",
"src/process.c",
"src/main.c"
]
}

18
src/controller/fetch.c
Unescape View File

@ -2,6 +2,7 @@
// Model
#include "model/status.h"
#include "model/account.h"
// Submodules
#include "ffdb/trie.h"
@ -59,10 +60,27 @@ static void process_fetch_once()
printf( "new count = %d\n", ffdb_trie_count(STUBS) );
}
static void process_account_sync_once()
{
for( int i = 0; i < 10; ++i ) {
struct account* a = account_load_next_update();
if( !a ) { continue; }
if( time(NULL) >= a->next_update ) {
account_sync_from_activity_pub( a->id );
}
account_free(a);
return;
}
}
void process_fetch()
{
while(true) {
process_fetch_once();
process_account_sync_once();
for( int i = 0; i < 30; ++i ) {
if( terminate ) { return; };

2
src/ffdb

@ -1 +1 @@
Subproject commit 7e76a21254e772b6bcc955fcb575b5f40a67dbb4
Subproject commit 06066e9de27b265e1191de272a67e5e4ba2353f8

85
src/model/account.c
Unescape View File

@ -1,16 +1,6 @@
#define _GNU_SOURCE
#include "account.h"
// Submodules
#include "json/json.h"
#include "json/layout.h"
#include "ffdb/fs_list.h"
#include "ffdb/hash_index.h"
#include "ffdb/trie.h"
#include "util/format.h"
#include "collections/array.h"
#include "ap/object.h"
// Model
#include "model/server.h"
#include "model/status.h"
@ -24,6 +14,17 @@
// View
#include "view/api/Relationship.h"
// Submodules
#include "json/json.h"
#include "json/layout.h"
#include "ffdb/fs_list.h"
#include "ffdb/hash_index.h"
#include "ffdb/trie.h"
#include "collections/array.h"
#include "ap/object.h"
#include "util/format.h"
#include "util/time.h"
// Stdlib
#include <stdio.h>
#include <stdlib.h>
@ -65,6 +66,7 @@ static struct json_object_field account_layout[] = {
JSON_FIELD_STRING( display_name, true ),
JSON_FIELD_BOOL( stub, false ),
JSON_FIELD_DATETIME( next_stub_recheck, false ),
JSON_FIELD_DATETIME( next_update, false ),
JSON_FIELD_STRING( avatar, false ),
{
.key = "avatar",
@ -450,12 +452,75 @@ void account_save( struct account* a )
json_write_object_layout_to_file( filename, "\t", account_layout, a );
// Index by uri
if( a->account_url ) {
char id_str[32];
snprintf( id_str,32, "%d", a->id );
ffdb_trie_set( "data/accounts/by-uri", a->account_url, id_str );
hash_index_remove( "data/accounts/uri_index/", a->account_url );
}
// Index by next_update
{
char timestamp[128];
rfc3339_time_string( a->next_update, timestamp, sizeof(timestamp) );
char key[512];
snprintf( key,512, "%s-%d", timestamp, a->id );
char id_str[32];
snprintf( id_str,32, "%d", a->id );
ffdb_trie_set( "data/accounts/updates/", key, id_str );
}
}
struct account* account_load_next_update()
{
int count = ffdb_trie_count( "data/accounts/updates/" );
if( count == 0 ) {
printf( "No accounts to update\n" );
return NULL;
}
char* key = NULL;
char* value = NULL;
count -= 1;
while( !ffdb_trie_get_index( "data/accounts/updates/", count, &key, &value ) ) {
if( !count ) {
return NULL;
}
printf( "Unable to load index entry #%d\n", count );
free(key); key = NULL;
free(value); value = NULL;
count -= 1;
}
struct account* result = account_from_id( atoi(value) );
if( !result ) {
printf( "Unable to load account with id=%s\n", value );
goto reindex;
}
// Check to make sure the index entry is valid
char timestamp[512];
rfc3339_time_string( result->next_update, timestamp, 512 );
char buffer[512];
snprintf( buffer,sizeof(buffer), "%s-%s", timestamp, value );
if( 0 != strcmp( buffer, key ) ) {
printf( "Mismatch between index and account data (%s != %s). Fixing\n", key, buffer );
goto reindex;
}
free(key);
free(value);
printf( "Next account to update: %s\n", result->account_url );
return result;
reindex:
// It isn't, fix the index and retry
ffdb_trie_set( "data/accounts/updates/", key, NULL );
account_free(result);
free(key);
free(value);
return NULL;
}

2
src/model/account.h
Unescape View File

@ -44,6 +44,7 @@ struct account
bool stub;
bool defunct;
time_t next_stub_recheck;
time_t next_update;
int replaced_by;
int account_type;
@ -100,6 +101,7 @@ struct account* account_from_webfinger( const char* handle, const char* default_
struct account* account_new();
struct account* account_from_uri_or_fetch( const char* uri );
struct account* account_fetch_from_uri( const char* uri );
struct account* account_load_next_update();
void account_free( struct account* a );
void account_save( struct account* a );

14
src/model/account/ap_sync.c
Unescape View File

@ -36,14 +36,19 @@ bool account_sync_from_activity_pub( unsigned int account_id )
{
bool result = false;
struct account* a = account_from_id(account_id);
struct ap_object* obj = NULL;
if( !a ) {
a = account_new();
a->id = account_id;
} else if( a->next_update > time(NULL) ) {
goto succeeded;
} else if( a->local ) {
goto succeeded;
}
struct ap_object* obj = account_get_activity_pub_data(a);
obj = account_get_activity_pub_data(a);
if( !obj ) {
printf( "Failed to load AP data\n" );
printf( "Failed to load AP data for %s (%d)\n", a->account_url, a->id );
goto failed;
}
@ -51,6 +56,7 @@ bool account_sync_from_activity_pub( unsigned int account_id )
// If we got here, this account can't be a stub any more
a->stub = false;
a->next_update = time(NULL) + (60*60*24*3) + (rand() % (60*60*24*2)); // Next update in 3-5 days
account_save(a);
account_index_webfinger(a);
@ -60,6 +66,10 @@ succeeded:
result = true;
goto cleanup;
cleanup:
if( a ) {
a->next_update = time(NULL) + (60*60*24*3) + (rand() % (60*60*24*2)); // Next update in 3-5 days
account_save(a);
}
ap_object_free(obj);
account_free(a);

7
src/model/activity.c
Unescape View File

@ -99,7 +99,7 @@ struct ap_object* activity_create_EmojiReact( struct status* s, const char* reac
struct ap_object* act = activity_new_local_activity();
activity_allocate_local_id(act);
act->id = aformat( "https://%s/activity/%d", g_server->domain, act->local_id );
act->actor = aformat( "https://%s/owner/actor", g_server->domain );
act->actor = strdup(a->account_url);
act->type = pleroma_EmojiReact;
act->content.content = safe_strdup(react);
act->published = time(NULL);
@ -221,16 +221,17 @@ failed:
struct ap_object* activity_create_Like( struct status* s )
{
struct account* a = account_from_id( s->account_id );
struct ap_object* act = activity_new_local_activity();
activity_allocate_local_id(act);
act->id = aformat( "https://%s/activity/%d", g_server->domain, act->local_id );
act->actor = aformat( "https://%s/owner/actor", g_server->domain );
act->actor = strdup(a->account_url);
act->type = ap_Like;
act->published = time(NULL);
act->object.tag = apaot_ref;
act->object.ref = strdup( s->url );
struct account* a = account_from_id( s->account_id );
char* to = aformat( "https://%s/owner/followers", g_server->domain );
array_append( &act->to, sizeof(to), &to );
to = strdup(a->account_url);

25
src/model/timeline.c
Unescape View File

@ -1,12 +1,16 @@
#include "timeline.h"
// Model
#include "model/status.h"
// Submodules
#include "json/json.h"
#include "json/layout.h"
#include "ffdb/trie.h"
#include "util/format.h"
#include "util/time.h"
#include "model/status.h"
// Standard Library
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
@ -78,23 +82,10 @@ int timeline_load_statuses( struct timeline* tl, int offset_from_head, int count
free(keys.items);
return result_count;
}
static void key_for_post( struct status* s, char* key, int sizeof_key )
{
struct tm gmtime_data;
gmtime_r( &s->published, &gmtime_data );
snprintf( key, sizeof_key, "%04d-%02d-%02dT%02d:%02d:%02dZ",
gmtime_data.tm_year + 1900,
gmtime_data.tm_mon + 1,
gmtime_data.tm_mday,
gmtime_data.tm_hour,
gmtime_data.tm_min,
gmtime_data.tm_sec
);
}
void timeline_add_post( struct timeline* tl, struct status* s )
{
char key[512];
key_for_post(s,key,sizeof(key));
rfc3339_time_string( s->published, key, sizeof(key) );
char value[32];
ffdb_trie_set( tl->path, key, format( value, sizeof(value), "%d", s->id ) );
@ -102,7 +93,7 @@ void timeline_add_post( struct timeline* tl, struct status* s )
void timeline_remove_post( struct timeline* tl, struct status* s )
{
char key[512];
key_for_post(s,key,sizeof(key));
rfc3339_time_string( s->published, key, sizeof(key) );
ffdb_trie_remove( tl->path, key );
}

2
src/util

@ -1 +1 @@
Subproject commit 24b374ef03befcff66499be372d089ade16e2381
Subproject commit e133427bb7da0224fc365252406024c207418863
Write Preview
Loading…
Cancel
Save